//=============================================================================
int Ifpack_CrsRiluk::Multiply(bool Trans, const Epetra_MultiVector& X,
Epetra_MultiVector& Y) const {
//
// This function finds X such that LDU Y = X or U(trans) D L(trans) Y = X for multiple RHS
//
// First generate X and Y as needed for this function
Teuchos::RefCountPtr<Epetra_MultiVector> X1;
Teuchos::RefCountPtr<Epetra_MultiVector> Y1;
EPETRA_CHK_ERR(GenerateXY(Trans, X, Y, &X1, &Y1));
#ifdef IFPACK_FLOPCOUNTERS
Epetra_Flops * counter = this->GetFlopCounter();
if (counter!=0) {
L_->SetFlopCounter(*counter);
Y1->SetFlopCounter(*counter);
U_->SetFlopCounter(*counter);
}
#endif
if (!Trans) {
EPETRA_CHK_ERR(U_->Multiply(Trans, *X1, *Y1)); //
EPETRA_CHK_ERR(Y1->Update(1.0, *X1, 1.0)); // Y1 = Y1 + X1 (account for implicit unit diagonal)
EPETRA_CHK_ERR(Y1->ReciprocalMultiply(1.0, *D_, *Y1, 0.0)); // y = D*y (D_ has inverse of diagonal)
Epetra_MultiVector Y1temp(*Y1); // Need a temp copy of Y1
EPETRA_CHK_ERR(L_->Multiply(Trans, Y1temp, *Y1));
EPETRA_CHK_ERR(Y1->Update(1.0, Y1temp, 1.0)); // (account for implicit unit diagonal)
if (IsOverlapped_) {EPETRA_CHK_ERR(Y.Export(*Y1,*L_->Exporter(), OverlapMode_));} // Export computed Y values if needed
}
else {
EPETRA_CHK_ERR(L_->Multiply(Trans, *X1, *Y1));
EPETRA_CHK_ERR(Y1->Update(1.0, *X1, 1.0)); // Y1 = Y1 + X1 (account for implicit unit diagonal)
EPETRA_CHK_ERR(Y1->ReciprocalMultiply(1.0, *D_, *Y1, 0.0)); // y = D*y (D_ has inverse of diagonal)
Epetra_MultiVector Y1temp(*Y1); // Need a temp copy of Y1
EPETRA_CHK_ERR(U_->Multiply(Trans, Y1temp, *Y1));
EPETRA_CHK_ERR(Y1->Update(1.0, Y1temp, 1.0)); // (account for implicit unit diagonal)
if (IsOverlapped_) {EPETRA_CHK_ERR(Y.Export(*Y1,*L_->Exporter(), OverlapMode_));}
}
return(0);
}
//=============================================================================
int Ifpack_CrsRiluk::Solve(bool Trans, const Epetra_MultiVector& X,
Epetra_MultiVector& Y) const {
//
// This function finds Y such that LDU Y = X or U(trans) D L(trans) Y = X for multiple RHS
//
// First generate X and Y as needed for this function
Teuchos::RefCountPtr<Epetra_MultiVector> X1;
Teuchos::RefCountPtr<Epetra_MultiVector> Y1;
EPETRA_CHK_ERR(GenerateXY(Trans, X, Y, &X1, &Y1));
bool Upper = true;
bool Lower = false;
bool UnitDiagonal = true;
#ifdef IFPACK_FLOPCOUNTERS
Epetra_Flops * counter = this->GetFlopCounter();
if (counter!=0) {
L_->SetFlopCounter(*counter);
Y1->SetFlopCounter(*counter);
U_->SetFlopCounter(*counter);
}
#endif
if (!Trans) {
EPETRA_CHK_ERR(L_->Solve(Lower, Trans, UnitDiagonal, *X1, *Y1));
EPETRA_CHK_ERR(Y1->Multiply(1.0, *D_, *Y1, 0.0)); // y = D*y (D_ has inverse of diagonal)
EPETRA_CHK_ERR(U_->Solve(Upper, Trans, UnitDiagonal, *Y1, *Y1)); // Solve Uy = y
if (IsOverlapped_) {EPETRA_CHK_ERR(Y.Export(*Y1,*L_->Exporter(), OverlapMode_));} // Export computed Y values if needed
}
else {
EPETRA_CHK_ERR(U_->Solve(Upper, Trans, UnitDiagonal, *X1, *Y1)); // Solve Uy = y
EPETRA_CHK_ERR(Y1->Multiply(1.0, *D_, *Y1, 0.0)); // y = D*y (D_ has inverse of diagonal)
EPETRA_CHK_ERR(L_->Solve(Lower, Trans, UnitDiagonal, *Y1, *Y1));
if (IsOverlapped_) {EPETRA_CHK_ERR(Y.Export(*Y1,*U_->Importer(), OverlapMode_));} // Export computed Y values if needed
}
return(0);
}
//=========================================================================
// Returns the result of a Epetra_Operator inverse applied to an Epetra_MultiVector X in Y.
int EpetraExt_PointToBlockDiagPermute::ApplyInverse(const Epetra_MultiVector& X, Epetra_MultiVector& Y) const{
// Stuff borrowed from Epetra_CrsMatrix
int NumVectors = X.NumVectors();
if (NumVectors!=Y.NumVectors()) {
EPETRA_CHK_ERR(-2); // Need same number of vectors in each MV
}
const Epetra_MultiVector *Xp=&X;
Epetra_MultiVector *Yp=&Y;
// Allocate temp workspace if X==Y and there are no imports or exports
Epetra_MultiVector * Xcopy = 0;
if (&X==&Y && Importer_==0 && Exporter_==0) {
Xcopy = new Epetra_MultiVector(X);
Xp=Xcopy;
}
UpdateImportVector(NumVectors); // Make sure Import and Export Vectors are compatible
UpdateExportVector(NumVectors);
// If we have a non-trivial importer, we must import elements that are permuted or are on other processors
if (Importer_){
EPETRA_CHK_ERR(ImportVector_->Import(X, *Importer_, Insert));
Xp=ImportVector_;
}
// If we have a non-trivial exporter, we must export elements that are permuted or belong to other processors
if (Exporter_) {
Yp=ExportVector_;
}
// Do the matvec
BDMat_->ApplyInverse(*Xp,*Yp);
// Export if needed
if (Exporter_) {
Y.PutScalar(0.0); // Make sure target is zero
Y.Export(*ExportVector_, *Exporter_, Add); // Fill Y with Values from export vector
}
// Cleanup
if(Xcopy) {
delete Xcopy;
EPETRA_CHK_ERR(1); // Return positive code to alert the user about needing extra copy of X
return 1;
}
return 0;
}
// collect subvectors into a single global vector
void many2one(Epetra_MultiVector & one, const std::vector<RCP<const Epetra_MultiVector> > & many,
const std::vector<RCP<Epetra_Export> > & subExport)
{
// std::vector<RCP<const Epetra_Vector> >::const_iterator vecItr;
std::vector<RCP<const Epetra_MultiVector> >::const_iterator vecItr;
std::vector<RCP<Epetra_Export> >::const_iterator expItr;
// using Exporters fill the empty vector from the sub-vectors
for(vecItr=many.begin(),expItr=subExport.begin();
vecItr!=many.end();++vecItr,++expItr) {
// for ease of access to the source
RCP<const Epetra_MultiVector> srcVec = *vecItr;
// extract the map with global indicies from the current vector
const Epetra_Map & globalMap = *(Teuchos::get_extra_data<RCP<Epetra_Map> >(srcVec,"globalMap"));
// build the export vector as a view of the destination
Epetra_MultiVector exportVector(View,globalMap,srcVec->Values(),srcVec->Stride(),srcVec->NumVectors());
one.Export(exportVector,**expItr,Insert);
}
}
//=============================================================================
int Amesos_Dscpack::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(0);
ResetTimer(1);
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
// MS // some checks on matrix size
if (RowMatrixA->NumGlobalRows() != RowMatrixA->NumGlobalCols())
AMESOS_CHK_ERR(-1);
// Convert vector b to a vector in the form that DSCPACK needs it
//
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong with input
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-2);
double *dscmapXvalues ;
int dscmapXlda ;
Epetra_MultiVector dscmapX(DscRowMap(),NumVectors) ;
int ierr;
AMESOS_CHK_ERR(dscmapX.ExtractView(&dscmapXvalues,&dscmapXlda));
assert (dscmapXlda == NumLocalCols);
double *dscmapBvalues ;
int dscmapBlda ;
Epetra_MultiVector dscmapB(DscRowMap(), NumVectors ) ;
ierr = dscmapB.ExtractView( &dscmapBvalues, &dscmapBlda );
AMESOS_CHK_ERR(ierr);
assert( dscmapBlda == NumLocalCols ) ;
AMESOS_CHK_ERR(dscmapB.Import(*vecB, Importer(), Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
ResetTimer(0);
// MS // now solve the problem
std::vector<double> ValuesInNewOrder( NumLocalCols ) ;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
if ( MyDscRank >= 0 ) {
for ( int j =0 ; j < NumVectors; j++ ) {
for ( int i = 0; i < NumLocalCols; i++ ) {
ValuesInNewOrder[i] = dscmapBvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapBlda ] ;
}
AMESOS_CHK_ERR( DSC_InputRhsLocalVec ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
AMESOS_CHK_ERR( DSC_Solve ( PrivateDscpackData_->MyDSCObject_ ) ) ;
AMESOS_CHK_ERR( DSC_GetLocalSolution ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
for ( int i = 0; i < NumLocalCols; i++ ) {
dscmapXvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapXlda ] = ValuesInNewOrder[i];
}
}
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
ResetTimer(0);
ResetTimer(1);
vecX->Export( dscmapX, Importer(), Insert ) ;
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
ComputeTrueResidual(*(GetProblem()->GetMatrix()), *vecX, *vecB,
false, "Amesos_Dscpack");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Dscpack");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
NumSolve_++;
return(0) ;
}
//==============================================================================
int LinearProblem_CrsSingletonFilter::UpdateReducedProblem(Epetra_LinearProblem * Problem) {
int i, j;
if (Problem==0) EPETRA_CHK_ERR(-1); // Null problem pointer
FullProblem_ = Problem;
FullMatrix_ = dynamic_cast<Epetra_RowMatrix *>(Problem->GetMatrix());
if (FullMatrix_==0) EPETRA_CHK_ERR(-2); // Need a RowMatrix
if (Problem->GetRHS()==0) EPETRA_CHK_ERR(-3); // Need a RHS
if (Problem->GetLHS()==0) EPETRA_CHK_ERR(-4); // Need a LHS
if (!HaveReducedProblem_) EPETRA_CHK_ERR(-5); // Must have set up reduced problem
// Create pointer to Full RHS, LHS
Epetra_MultiVector * FullRHS = FullProblem()->GetRHS();
Epetra_MultiVector * FullLHS = FullProblem()->GetLHS();
int NumVectors = FullLHS->NumVectors();
int NumEntries;
int * Indices;
double * Values;
int NumMyRows = FullMatrix()->NumMyRows();
int ColSingletonCounter = 0;
for (i=0; i<NumMyRows; i++) {
int curGRID = FullMatrixRowMap().GID(i);
if (ReducedMatrixRowMap()->MyGID(curGRID)) { // Check if this row should go into reduced matrix
EPETRA_CHK_ERR(GetRowGCIDs(i, NumEntries, Values, Indices)); // Get current row (indices global)
int ierr = ReducedMatrix()->ReplaceGlobalValues(curGRID, NumEntries,
Values, Indices);
// Positive errors will occur because we are submitting col entries that are not part of
// reduced system. However, because we specified a column map to the ReducedMatrix constructor
// these extra column entries will be ignored and we will be politely reminded by a positive
// error code
if (ierr<0) EPETRA_CHK_ERR(ierr);
}
// Otherwise if singleton row we explicitly eliminate this row and solve for corresponding X value
else {
EPETRA_CHK_ERR(GetRow(i, NumEntries, Values, Indices)); // Get current row
if (NumEntries==1) {
double pivot = Values[0];
if (pivot==0.0) EPETRA_CHK_ERR(-1); // Encountered zero row, unable to continue
int indX = Indices[0];
for (j=0; j<NumVectors; j++)
(*tempExportX_)[j][indX] = (*FullRHS)[j][i]/pivot;
}
// Otherwise, this is a singleton column and we will scan for the pivot element needed
// for post-solve equations
else {
j = ColSingletonPivotLIDs_[ColSingletonCounter];
double pivot = Values[j];
if (pivot==0.0) EPETRA_CHK_ERR(-2); // Encountered zero column, unable to continue
ColSingletonPivots_[ColSingletonCounter] = pivot;
ColSingletonCounter++;
}
}
}
assert(ColSingletonCounter==NumMyColSingletons_); // Sanity test
// Update Reduced LHS (Puts any initial guess values into reduced system)
ReducedLHS_->PutScalar(0.0); // zero out Reduced LHS
EPETRA_CHK_ERR(ReducedLHS_->Import(*FullLHS, *Full2ReducedLHSImporter_, Insert));
FullLHS->PutScalar(0.0); // zero out Full LHS since we will inject values as we get them
// Construct Reduced RHS
// Zero out temp space
tempX_->PutScalar(0.0);
tempB_->PutScalar(0.0);
//Inject known X values into tempX for purpose of computing tempB = FullMatrix*tempX
// Also inject into full X since we already know the solution
if (FullMatrix()->RowMatrixImporter()!=0) {
EPETRA_CHK_ERR(tempX_->Export(*tempExportX_, *FullMatrix()->RowMatrixImporter(), Add));
EPETRA_CHK_ERR(FullLHS->Export(*tempExportX_, *FullMatrix()->RowMatrixImporter(), Add));
}
else {
tempX_->Update(1.0, *tempExportX_, 0.0);
FullLHS->Update(1.0, *tempExportX_, 0.0);
}
EPETRA_CHK_ERR(FullMatrix()->Multiply(false, *tempX_, *tempB_));
EPETRA_CHK_ERR(tempB_->Update(1.0, *FullRHS, -1.0)); // tempB now has influence of already-known X values
ReducedRHS_->PutScalar(0.0);
EPETRA_CHK_ERR(ReducedRHS_->Import(*tempB_, *Full2ReducedRHSImporter_, Insert));
return(0);
}
//==============================================================================
int LinearProblem_CrsSingletonFilter::ConstructReducedProblem(Epetra_LinearProblem * Problem) {
int i, j;
if (HaveReducedProblem_) EPETRA_CHK_ERR(-1); // Setup already done once. Cannot do it again
if (Problem==0) EPETRA_CHK_ERR(-2); // Null problem pointer
FullProblem_ = Problem;
FullMatrix_ = dynamic_cast<Epetra_RowMatrix *>(Problem->GetMatrix());
if (FullMatrix_==0) EPETRA_CHK_ERR(-3); // Need a RowMatrix
if (Problem->GetRHS()==0) EPETRA_CHK_ERR(-4); // Need a RHS
if (Problem->GetLHS()==0) EPETRA_CHK_ERR(-5); // Need a LHS
// Generate reduced row and column maps
Epetra_MapColoring & RowMapColors = *RowMapColors_;
Epetra_MapColoring & ColMapColors = *ColMapColors_;
ReducedMatrixRowMap_ = RowMapColors.GenerateMap(0);
ReducedMatrixColMap_ = ColMapColors.GenerateMap(0);
// Create domain and range map colorings by exporting map coloring of column and row maps
if (FullMatrix()->RowMatrixImporter()!=0) {
Epetra_MapColoring DomainMapColors(FullMatrixDomainMap());
EPETRA_CHK_ERR(DomainMapColors.Export(*ColMapColors_, *FullMatrix()->RowMatrixImporter(), AbsMax));
OrigReducedMatrixDomainMap_ = DomainMapColors.GenerateMap(0);
}
else
OrigReducedMatrixDomainMap_ = ReducedMatrixColMap_;
if (FullMatrixIsCrsMatrix_) {
if (FullCrsMatrix()->Exporter()!=0) { // Non-trivial exporter
Epetra_MapColoring RangeMapColors(FullMatrixRangeMap());
EPETRA_CHK_ERR(RangeMapColors.Export(*RowMapColors_, *FullCrsMatrix()->Exporter(),
AbsMax));
ReducedMatrixRangeMap_ = RangeMapColors.GenerateMap(0);
}
else
ReducedMatrixRangeMap_ = ReducedMatrixRowMap_;
}
else
ReducedMatrixRangeMap_ = ReducedMatrixRowMap_;
// Check to see if the reduced system domain and range maps are the same.
// If not, we need to remap entries of the LHS multivector so that they are distributed
// conformally with the rows of the reduced matrix and the RHS multivector
SymmetricElimination_ = ReducedMatrixRangeMap_->SameAs(*OrigReducedMatrixDomainMap_);
if (!SymmetricElimination_)
ConstructRedistributeExporter(OrigReducedMatrixDomainMap_, ReducedMatrixRangeMap_,
RedistributeDomainExporter_, ReducedMatrixDomainMap_);
else {
ReducedMatrixDomainMap_ = OrigReducedMatrixDomainMap_;
OrigReducedMatrixDomainMap_ = 0;
RedistributeDomainExporter_ = 0;
}
// Create pointer to Full RHS, LHS
Epetra_MultiVector * FullRHS = FullProblem()->GetRHS();
Epetra_MultiVector * FullLHS = FullProblem()->GetLHS();
int NumVectors = FullLHS->NumVectors();
// Create importers
// cout << "RedDomainMap\n";
// cout << *ReducedMatrixDomainMap();
// cout << "FullDomainMap\n";
// cout << FullMatrixDomainMap();
Full2ReducedLHSImporter_ = new Epetra_Import(*ReducedMatrixDomainMap(), FullMatrixDomainMap());
// cout << "RedRowMap\n";
// cout << *ReducedMatrixRowMap();
// cout << "FullRHSMap\n";
// cout << FullRHS->Map();
Full2ReducedRHSImporter_ = new Epetra_Import(*ReducedMatrixRowMap(), FullRHS->Map());
// Construct Reduced Matrix
ReducedMatrix_ = new Epetra_CrsMatrix(Copy, *ReducedMatrixRowMap(), *ReducedMatrixColMap(), 0);
// Create storage for temporary X values due to explicit elimination of rows
tempExportX_ = new Epetra_MultiVector(FullMatrixColMap(), NumVectors);
int NumEntries;
int * Indices;
double * Values;
int NumMyRows = FullMatrix()->NumMyRows();
int ColSingletonCounter = 0;
for (i=0; i<NumMyRows; i++) {
int curGRID = FullMatrixRowMap().GID(i);
if (ReducedMatrixRowMap()->MyGID(curGRID)) { // Check if this row should go into reduced matrix
EPETRA_CHK_ERR(GetRowGCIDs(i, NumEntries, Values, Indices)); // Get current row (Indices are global)
int ierr = ReducedMatrix()->InsertGlobalValues(curGRID, NumEntries,
Values, Indices); // Insert into reduce matrix
// Positive errors will occur because we are submitting col entries that are not part of
// reduced system. However, because we specified a column map to the ReducedMatrix constructor
// these extra column entries will be ignored and we will be politely reminded by a positive
// error code
if (ierr<0) EPETRA_CHK_ERR(ierr);
}
else {
EPETRA_CHK_ERR(GetRow(i, NumEntries, Values, Indices)); // Get current row
if (NumEntries==1) {
double pivot = Values[0];
if (pivot==0.0) EPETRA_CHK_ERR(-1); // Encountered zero row, unable to continue
int indX = Indices[0];
for (j=0; j<NumVectors; j++)
(*tempExportX_)[j][indX] = (*FullRHS)[j][i]/pivot;
}
// Otherwise, this is a singleton column and we will scan for the pivot element needed
// for post-solve equations
else {
int targetCol = ColSingletonColLIDs_[ColSingletonCounter];
for (j=0; j<NumEntries; j++) {
if (Indices[j]==targetCol) {
double pivot = Values[j];
if (pivot==0.0) EPETRA_CHK_ERR(-2); // Encountered zero column, unable to continue
ColSingletonPivotLIDs_[ColSingletonCounter] = j; // Save for later use
ColSingletonPivots_[ColSingletonCounter] = pivot;
ColSingletonCounter++;
break;
}
}
}
}
}
// Now convert to local indexing. We have constructed things so that the domain and range of the
// matrix will have the same map. If the reduced matrix domain and range maps were not the same, the
// differences were addressed in the ConstructRedistributeExporter() method
EPETRA_CHK_ERR(ReducedMatrix()->FillComplete(*ReducedMatrixDomainMap(), *ReducedMatrixRangeMap()));
// Construct Reduced LHS (Puts any initial guess values into reduced system)
ReducedLHS_ = new Epetra_MultiVector(*ReducedMatrixDomainMap(), NumVectors);
EPETRA_CHK_ERR(ReducedLHS_->Import(*FullLHS, *Full2ReducedLHSImporter_, Insert));
FullLHS->PutScalar(0.0); // zero out Full LHS since we will inject values as we get them
// Construct Reduced RHS
// First compute influence of already-known values of X on RHS
tempX_ = new Epetra_MultiVector(FullMatrixDomainMap(), NumVectors);
tempB_ = new Epetra_MultiVector(FullRHS->Map(), NumVectors);
//Inject known X values into tempX for purpose of computing tempB = FullMatrix*tempX
// Also inject into full X since we already know the solution
if (FullMatrix()->RowMatrixImporter()!=0) {
EPETRA_CHK_ERR(tempX_->Export(*tempExportX_, *FullMatrix()->RowMatrixImporter(), Add));
EPETRA_CHK_ERR(FullLHS->Export(*tempExportX_, *FullMatrix()->RowMatrixImporter(), Add));
}
else {
tempX_->Update(1.0, *tempExportX_, 0.0);
FullLHS->Update(1.0, *tempExportX_, 0.0);
}
EPETRA_CHK_ERR(FullMatrix()->Multiply(false, *tempX_, *tempB_));
EPETRA_CHK_ERR(tempB_->Update(1.0, *FullRHS, -1.0)); // tempB now has influence of already-known X values
ReducedRHS_ = new Epetra_MultiVector(*ReducedMatrixRowMap(), FullRHS->NumVectors());
EPETRA_CHK_ERR(ReducedRHS_->Import(*tempB_, *Full2ReducedRHSImporter_, Insert));
// Finally construct Reduced Linear Problem
ReducedProblem_ = new Epetra_LinearProblem(ReducedMatrix_, ReducedLHS_, ReducedRHS_);
double fn = FullMatrix()->NumGlobalRows();
double fnnz = FullMatrix()->NumGlobalNonzeros();
double rn = ReducedMatrix()->NumGlobalRows();
double rnnz = ReducedMatrix()->NumGlobalNonzeros();
RatioOfDimensions_ = rn/fn;
RatioOfNonzeros_ = rnnz/fnnz;
HaveReducedProblem_ = true;
return(0);
}
//=============================================================================
int Epetra_FastCrsMatrix::Multiply(bool TransA, const Epetra_MultiVector& X, Epetra_MultiVector& Y) const {
//
// This function forms the product Y = A * Y or Y = A' * X
//
if (X.NumVectors()==1 && Y.NumVectors()==1) {
double * xp = (double *) X[0];
double * yp = (double *) Y[0];
Epetra_Vector x(View, X.Map(), xp);
Epetra_Vector y(View, Y.Map(), yp);
return(Multiply(TransA, x, y));
}
if (!Filled()) EPETRA_CHK_ERR(-1); // Matrix must be filled.
int i, j, k;
int * NumEntriesPerRow = NumEntriesPerRow_;
int ** Indices = Indices_;
double ** Values = Values_;
double **Xp = (double**)X.Pointers();
double **Yp = (double**)Y.Pointers();
int NumVectors = X.NumVectors();
int NumMyCols_ = NumMyCols();
// Need to better manage the Import and Export vectors:
// - Need accessor functions
// - Need to make the NumVector match (use a View to do this)
// - Need to look at RightScale and ColSum routines too.
if (!TransA) {
// If we have a non-trivial importer, we must import elements that are permuted or are on other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=NumVectors) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),NumVectors); // Create import vector if needed
ImportVector_->Import(X, *Importer(), Insert);
Xp = (double**)ImportVector_->Pointers();
}
// If we have a non-trivial exporter, we must export elements that are permuted or belong to other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=NumVectors) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),NumVectors); // Create Export vector if needed
Yp = (double**)ExportVector_->Pointers();
}
// Do actual computation
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (k=0; k<NumVectors; k++) {
double sum = 0.0;
for (j=0; j < NumEntries; j++) sum += RowValues[j] * Xp[k][RowIndices[j]];
Yp[k][i] = sum;
}
}
if (Exporter()!=0) Y.Export(*ExportVector_, *Exporter(), Add); // Fill Y with Values from export vector
}
else { // Transpose operation
// If we have a non-trivial exporter, we must import elements that are permuted or are on other processors
if (Exporter()!=0) {
if (ExportVector_!=0) {
if (ExportVector_->NumVectors()!=NumVectors) { delete ExportVector_; ExportVector_= 0;}
}
if (ExportVector_==0) ExportVector_ = new Epetra_MultiVector(RowMap(),NumVectors); // Create Export vector if needed
ExportVector_->Import(X, *Exporter(), Insert);
Xp = (double**)ExportVector_->Pointers();
}
// If we have a non-trivial importer, we must export elements that are permuted or belong to other processors
if (Importer()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=NumVectors) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(ColMap(),NumVectors); // Create import vector if needed
Yp = (double**)ImportVector_->Pointers();
}
// Do actual computation
for (k=0; k<NumVectors; k++)
for (i=0; i < NumMyCols_; i++) Yp[k][i] = 0.0; // Initialize y for transpose multiply
for (i=0; i < NumMyRows_; i++) {
int NumEntries = *NumEntriesPerRow++;
int * RowIndices = *Indices++;
double * RowValues = *Values++;
for (k=0; k<NumVectors; k++) {
for (j=0; j < NumEntries; j++) Yp[k][RowIndices[j]] += RowValues[j] * Xp[k][i];
}
}
if (Importer()!=0) Y.Export(*ImportVector_, *Importer(), Add); // Fill Y with Values from export vector
}
UpdateFlops(2*NumVectors*NumGlobalNonzeros64());
return(0);
}
//=============================================================================
int Amesos_Umfpack::Solve()
{
// if necessary, perform numeric factorization.
// This may call SymbolicFactorization() as well.
if (!IsNumericFactorizationOK_)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
Epetra_MultiVector *SerialB, *SerialX;
// Extract Serial versions of X and B
//
double *SerialXvalues ;
double *SerialBvalues ;
Epetra_MultiVector* SerialXextract = 0;
Epetra_MultiVector* SerialBextract = 0;
// Copy B to the serial version of B
//
ResetTimer(0);
if (IsLocal_ == 1) {
SerialB = vecB ;
SerialX = vecX ;
} else {
assert (IsLocal_ == 0);
SerialXextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract->Import(*vecB,Importer(),Insert);
SerialB = SerialBextract;
SerialX = SerialXextract;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call UMFPACK to perform the solve
// Note: UMFPACK uses a Compressed Column Storage instead of compressed row storage,
// Hence to compute A X = B, we ask UMFPACK to perform A^T X = B and vice versa
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
ResetTimer(0);
int SerialBlda, SerialXlda ;
int UmfpackRequest = UseTranspose()?UMFPACK_A:UMFPACK_At ;
int status = 0;
if ( MyPID_ == 0 ) {
int ierr;
ierr = SerialB->ExtractView(&SerialBvalues, &SerialBlda);
assert (ierr == 0);
ierr = SerialX->ExtractView(&SerialXvalues, &SerialXlda);
assert (ierr == 0);
assert( SerialBlda == NumGlobalElements_ ) ;
assert( SerialXlda == NumGlobalElements_ ) ;
for ( int j =0 ; j < NumVectors; j++ ) {
double *Control = (double *) NULL, *Info = (double *) NULL ;
status = umfpack_di_solve (UmfpackRequest, &Ap[0],
&Ai[0], &Aval[0],
&SerialXvalues[j*SerialXlda],
&SerialBvalues[j*SerialBlda],
Numeric, Control, Info) ;
}
}
if (status) AMESOS_CHK_ERR(status);
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if ( IsLocal_ == 0 ) {
vecX->Export(*SerialX, Importer(), Insert ) ;
if (SerialBextract) delete SerialBextract ;
if (SerialXextract) delete SerialXextract ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
{
Epetra_RowMatrix* Matrix =
dynamic_cast<Epetra_RowMatrix*>(Problem_->GetOperator());
ComputeTrueResidual(*Matrix, *vecX, *vecB, UseTranspose(), "Amesos_Umfpack");
}
if (ComputeVectorNorms_) {
ComputeVectorNorms(*vecX, *vecB, "Amesos_Umfpack");
}
NumSolve_++;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // Amesos overhead
return(0);
}
//=============================================================================
int Amesos_Klu::Solve()
{
Epetra_MultiVector* vecX = 0 ;
Epetra_MultiVector* vecB = 0 ;
#ifdef HAVE_AMESOS_EPETRAEXT
Teuchos::RCP<Epetra_MultiVector> vecX_rcp;
Teuchos::RCP<Epetra_MultiVector> vecB_rcp;
#endif
#ifdef Bug_8212
// This demonstrates Bug #2812 - Valgrind does not catch this
// memory leak
lose_this_ = (int *) malloc( 300 ) ;
#ifdef Bug_8212_B
// This demonstrates Bug #2812 - Valgrind does catch this
// use of unitialized data - but only in TestOptions/TestOptions.exe
// not in Test_Basic/amesos_test.exe
//
if ( lose_this_[0] == 12834 ) {
std::cout << __FILE__ << "::" << __LINE__ << " very unlikely to happen " << std::endl ;
}
#endif
#endif
if ( !TrustMe_ ) {
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false);
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false);
Epetra_MultiVector* OrigVecX ;
Epetra_MultiVector* OrigVecB ;
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
//
// Reindex the LHS and RHS
//
OrigVecX = Problem_->GetLHS() ;
OrigVecB = Problem_->GetRHS() ;
if ( Reindex_ ) {
#ifdef HAVE_AMESOS_EPETRAEXT
vecX_rcp = StdIndexDomain_->StandardizeIndex( *OrigVecX ) ;
vecB_rcp = StdIndexRange_->StandardizeIndex( *OrigVecB ) ;
vecX = &*vecX_rcp;
vecB = &*vecB_rcp;
#else
AMESOS_CHK_ERR( -13 ) ; // Amesos_Klu can't handle non-standard indexing without EpetraExt
#endif
} else {
vecX = OrigVecX ;
vecB = OrigVecB ;
}
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong in input
// Extract Serial versions of X and B
ResetTimer(0);
// Copy B to the serial version of B
//
if (UseDataInPlace_ == 1) {
#ifdef HAVE_AMESOS_EPETRAEXT
if(vecX_rcp==Teuchos::null)
SerialX_ = Teuchos::rcp(vecX,false);
else
SerialX_ = vecX_rcp;
if(vecB_rcp==Teuchos::null)
SerialB_ = Teuchos::rcp(vecB,false);
else
SerialB_ = vecB_rcp;
#else
SerialB_ = Teuchos::rcp(vecB,false);
SerialX_ = Teuchos::rcp(vecX,false);
#endif
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
} else {
assert (UseDataInPlace_ == 0);
if( NumVectors_ != Problem_->GetRHS()->NumVectors() ) {
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
}
if (NumVectors_ != vecB->NumVectors())
AMESOS_CHK_ERR(-1); // internal error
//ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecB->Map() ) );
//if ( SerialBextract_->Import(*vecB,*ImportRangeToSerial_,Insert) )
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportRangeToSerial_;
else UseImport=&*ImportDomainToSerial_;
if ( SerialBextract_->Import(*vecB,*UseImport,Insert) )
AMESOS_CHK_ERR( -1 ) ; // internal error
SerialB_ = Teuchos::rcp(&*SerialBextract_,false) ;
SerialX_ = Teuchos::rcp(&*SerialXextract_,false) ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call KLU to perform the solve
ResetTimer(0);
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
if (SerialXlda_ != NumGlobalElements_)
AMESOS_CHK_ERR(-1);
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
else
{
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false) ;
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false) ;
NumVectors_ = SerialX_->NumVectors();
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
}
}
if ( MyPID_ == 0) {
if ( NumVectors_ == 1 ) {
for ( int i = 0 ; i < NumGlobalElements_ ; i++ )
SerialXBvalues_[i] = SerialBvalues_[i] ;
} else {
SerialX_->Scale(1.0, *SerialB_ ) ; // X = B (Klu overwrites B with X)
}
if (UseTranspose()) {
amesos_klu_solve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
} else {
amesos_klu_tsolve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
}
}
if ( !TrustMe_ ) {
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if (UseDataInPlace_ == 0) {
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportDomainToSerial_;
else UseImport=&*ImportRangeToSerial_;
// ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecX->Map() ) );
vecX->Export( *SerialX_, *UseImport, Insert ) ;
} // otherwise we are already in place.
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
#if 0
//
// ComputeTrueResidual causes TestOptions to fail on my linux box
// Bug #1417
if (ComputeTrueResidual_)
ComputeTrueResidual(*SerialMatrix_, *vecX, *vecB, UseTranspose(), "Amesos_Klu");
#endif
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Klu");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
++NumSolve_;
return(0) ;
}
//=========================================================================
int Epetra_LinearProblemRedistor::CreateRedistProblem(const bool ConstructTranspose,
const bool MakeDataContiguous,
Epetra_LinearProblem *& RedistProblem) {
if (RedistProblemCreated_) EPETRA_CHK_ERR(-1); // This method can only be called once
Epetra_RowMatrix * OrigMatrix = OrigProblem_->GetMatrix();
Epetra_MultiVector * OrigLHS = OrigProblem_->GetLHS();
Epetra_MultiVector * OrigRHS = OrigProblem_->GetRHS();
if (OrigMatrix==0) EPETRA_CHK_ERR(-2); // There is no matrix associated with this Problem
if (RedistMap_==0) {
EPETRA_CHK_ERR(GenerateRedistMap());
}
RedistExporter_ = new Epetra_Export(OrigProblem_->GetMatrix()->RowMatrixRowMap(), *RedistMap_);
RedistProblem_ = new Epetra_LinearProblem();
Epetra_CrsMatrix * RedistMatrix;
// Check if the tranpose should be create or not
if (ConstructTranspose) {
Transposer_ = new Epetra_RowMatrixTransposer(OrigMatrix);
EPETRA_CHK_ERR(Transposer_->CreateTranspose(MakeDataContiguous, RedistMatrix, RedistMap_));
}
else {
// If not, then just do the redistribution based on the the RedistMap
RedistMatrix = new Epetra_CrsMatrix(Copy, *RedistMap_, 0);
// need to do this next step until we generalize the Import/Export ops for CrsMatrix
Epetra_CrsMatrix * OrigCrsMatrix = dynamic_cast<Epetra_CrsMatrix *>(OrigMatrix);
EPETRA_CHK_ERR(RedistMatrix->Export(*OrigCrsMatrix, *RedistExporter_, Add));
EPETRA_CHK_ERR(RedistMatrix->FillComplete());
}
RedistProblem_->SetOperator(RedistMatrix);
// Now redistribute the RHS and LHS if non-zero
Epetra_MultiVector * RedistLHS = 0;
Epetra_MultiVector * RedistRHS = 0;
int ierr = 0;
if (OrigLHS!=0) {
RedistLHS = new Epetra_MultiVector(*RedistMap_, OrigLHS->NumVectors());
EPETRA_CHK_ERR(RedistLHS->Export(*OrigLHS, *RedistExporter_, Add));
}
else ierr = 1;
if (OrigRHS!=0) {
RedistRHS = new Epetra_MultiVector(*RedistMap_, OrigLHS->NumVectors());
EPETRA_CHK_ERR(RedistRHS->Export(*OrigRHS, *RedistExporter_, Add));
}
else ierr ++;
RedistProblem_->SetLHS(RedistLHS);
RedistProblem_->SetRHS(RedistRHS);
RedistProblemCreated_ = true;
return(ierr);
}
int Amesos_Mumps::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
int NumVectors = vecX->NumVectors();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
if (Comm().NumProc() == 1)
{
// do not import any data
for (int j = 0 ; j < NumVectors; j++)
{
ResetTimer();
MDS.job = 3; // Request solve
for (int i = 0 ; i < Matrix().NumMyRows() ; ++i)
(*vecX)[j][i] = (*vecB)[j][i];
MDS.rhs = (*vecX)[j];
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
}
else
{
Epetra_MultiVector SerialVector(SerialMap(),NumVectors);
ResetTimer();
AMESOS_CHK_ERR(SerialVector.Import(*vecB,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
for (int j = 0 ; j < NumVectors; j++)
{
if (Comm().MyPID() == 0)
MDS.rhs = SerialVector[j];
// solve the linear system and take time
MDS.job = 3;
ResetTimer();
if (Comm().MyPID() < MaxProcs_)
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
// ship solution back and take timing
ResetTimer();
AMESOS_CHK_ERR(vecX->Export(SerialVector,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
}
if (ComputeTrueResidual_)
ComputeTrueResidual(Matrix(), *vecX, *vecB, UseTranspose(), "Amesos_Mumps");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Mumps");
NumSolve_++;
return(0) ;
}
int shylu_local_solve<Epetra_CrsMatrix, Epetra_MultiVector>
(
shylu_symbolic<Epetra_CrsMatrix,Epetra_MultiVector> *ssym,
shylu_data<Epetra_CrsMatrix,Epetra_MultiVector> *data,
shylu_config<Epetra_CrsMatrix,Epetra_MultiVector> *config,
const Epetra_MultiVector& X,
Epetra_MultiVector& Y
)
{
int err;
#ifndef NDEBUG
int nvectors = X.NumVectors();
assert (nvectors == data->localrhs->NumVectors());
#endif // NDEBUG
// Initialize the X vector for iterative solver
data->Xs->PutScalar(0.0);
// Get local portion of X
data->localrhs->Import(X, *(data->BdImporter), Insert);
data->localrhs->Print(std::cout);
std::cout << " " << std::endl;
data->locallhs->Print(std::cout);
// locallhs is z in paper
if (config->amesosForDiagonal) {
std::cout << "calling amesos for diagon" << endl;
ssym->OrigLP->SetRHS((data->localrhs).getRawPtr());
ssym->OrigLP->SetLHS((data->locallhs).getRawPtr());
std::cout << "set RHS and LHS " << std::endl;
ssym->ReIdx_LP->fwd();
ssym->Solver->Solve();
}
else {
ssym->ifSolver->ApplyInverse(*(data->localrhs), *(data->locallhs));
}
err = ssym->R->Multiply(false, *(data->locallhs), *(data->temp1));
assert (err == 0);
// Export temp1 to a dist vector - temp2
data->temp2->Import(*(data->temp1), *(data->DistImporter), Insert);
//Epetra_MultiVector Bs(SMap, nvectors); // b_2 - R * z in ShyLU paper
data->Bs->Import(X, *(data->BsImporter), Insert);
data->Bs->Update(-1.0, *(data->temp2), 1.0);
AztecOO *solver = 0;
Epetra_LinearProblem Problem(data->Sbar.get(),
(data->Xs).getRawPtr(), (data->Bs).getRawPtr());
if ((config->schurSolver == "G") || (config->schurSolver == "IQR"))
{
IFPACK_CHK_ERR(data->iqrSolver->Solve(*(data->schur_op),
*(data->Bs), *(data->Xs)));
}
else if (config->schurSolver == "Amesos")
{
Amesos_BaseSolver *solver2 = data->dsolver;
data->OrigLP2->SetLHS((data->Xs).getRawPtr());
data->OrigLP2->SetRHS((data->Bs).getRawPtr());
data->ReIdx_LP2->fwd();
//cout << "Calling solve *****************************" << endl;
solver2->Solve();
//cout << "Out of solve *****************************" << endl;
}
else
{
if (config->libName == "Belos")
{
solver = data->innersolver;
solver->SetLHS((data->Xs).getRawPtr());
solver->SetRHS((data->Bs).getRawPtr());
}
else
{
// See the comment above on why we are not able to reuse the solver
// when outer solve is AztecOO as well.
solver = new AztecOO();
//solver.SetPrecOperator(precop_);
solver->SetAztecOption(AZ_solver, AZ_gmres);
// Do not use AZ_none
solver->SetAztecOption(AZ_precond, AZ_dom_decomp);
//solver->SetAztecOption(AZ_precond, AZ_none);
//solver->SetAztecOption(AZ_precond, AZ_Jacobi);
////solver->SetAztecOption(AZ_precond, AZ_Neumann);
//solver->SetAztecOption(AZ_overlap, 3);
//solver->SetAztecOption(AZ_subdomain_solve, AZ_ilu);
//solver->SetAztecOption(AZ_output, AZ_all);
//solver->SetAztecOption(AZ_diagnostics, AZ_all);
solver->SetProblem(Problem);
}
// What should be a good inner_tolerance :-) ?
solver->Iterate(config->inner_maxiters, config->inner_tolerance);
}
// Import Xs locally
data->LocalXs->Import(*(data->Xs), *(data->XsImporter), Insert);
err = ssym->C->Multiply(false, *(data->LocalXs), *(data->temp3));
assert (err == 0);
data->temp3->Update(1.0, *(data->localrhs), -1.0);
if (config->amesosForDiagonal) {
ssym->OrigLP->SetRHS((data->temp3).getRawPtr());
ssym->OrigLP->SetLHS((data->locallhs).getRawPtr());
ssym->ReIdx_LP->fwd();
ssym->Solver->Solve();
}
else {
ssym->ifSolver->ApplyInverse(*(data->temp3), *(data->locallhs));
}
Y.Export(*(data->locallhs), *(data->XdExporter), Insert);
Y.Export(*(data->LocalXs), *(data->XsExporter), Insert);
if (config->libName == "Belos" || config->schurSolver == "Amesos")
{
// clean up
}
else
{
delete solver;
}
return 0;
}
int shylu_dist_solve<Epetra_CrsMatrix,Epetra_MultiVector>(
shylu_symbolic<Epetra_CrsMatrix,Epetra_MultiVector> *ssym,
shylu_data<Epetra_CrsMatrix,Epetra_MultiVector> *data,
shylu_config<Epetra_CrsMatrix,Epetra_MultiVector> *config,
const Epetra_MultiVector& X,
Epetra_MultiVector& Y
)
{
int err;
AztecOO *solver = 0;
assert(X.Map().SameAs(Y.Map()));
//assert(X.Map().SameAs(A_->RowMap()));
const Epetra_MultiVector *newX;
newX = &X;
//rd_->redistribute(X, newX);
int nvectors = newX->NumVectors();
// May have to use importer/exporter
Epetra_Map BsMap(-1, data->Snr, data->SRowElems, 0, X.Comm());
Epetra_Map BdMap(-1, data->Dnr, data->DRowElems, 0, X.Comm());
Epetra_MultiVector Bs(BsMap, nvectors);
Epetra_Import BsImporter(BsMap, newX->Map());
assert(BsImporter.SourceMap().SameAs(newX->Map()));
assert((newX->Map()).SameAs(BsImporter.SourceMap()));
Bs.Import(*newX, BsImporter, Insert);
Epetra_MultiVector Xs(BsMap, nvectors);
Epetra_SerialComm LComm; // Use Serial Comm for the local vectors.
Epetra_Map LocalBdMap(-1, data->Dnr, data->DRowElems, 0, LComm);
Epetra_MultiVector localrhs(LocalBdMap, nvectors);
Epetra_MultiVector locallhs(LocalBdMap, nvectors);
Epetra_MultiVector Z(BdMap, nvectors);
Epetra_MultiVector Bd(BdMap, nvectors);
Epetra_Import BdImporter(BdMap, newX->Map());
assert(BdImporter.SourceMap().SameAs(newX->Map()));
assert((newX->Map()).SameAs(BdImporter.SourceMap()));
Bd.Import(*newX, BdImporter, Insert);
int lda;
double *values;
err = Bd.ExtractView(&values, &lda);
assert (err == 0);
int nrows = ssym->C->RowMap().NumMyElements();
// copy to local vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = localrhs.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
// TODO : Do we need to reset the lhs and rhs here ?
if (config->amesosForDiagonal)
{
ssym->LP->SetRHS(&localrhs);
ssym->LP->SetLHS(&locallhs);
ssym->Solver->Solve();
}
else
{
ssym->ifSolver->ApplyInverse(localrhs, locallhs);
}
err = locallhs.ExtractView(&values, &lda);
assert (err == 0);
// copy to distributed vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = Z.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
Epetra_MultiVector temp1(BsMap, nvectors);
ssym->R->Multiply(false, Z, temp1);
Bs.Update(-1.0, temp1, 1.0);
Xs.PutScalar(0.0);
Epetra_LinearProblem Problem(data->Sbar.get(), &Xs, &Bs);
if (config->schurSolver == "Amesos")
{
Amesos_BaseSolver *solver2 = data->dsolver;
data->LP2->SetLHS(&Xs);
data->LP2->SetRHS(&Bs);
//cout << "Calling solve *****************************" << endl;
solver2->Solve();
//cout << "Out of solve *****************************" << endl;
}
else
{
if (config->libName == "Belos")
{
solver = data->innersolver;
solver->SetLHS(&Xs);
solver->SetRHS(&Bs);
}
else
{
// See the comment above on why we are not able to reuse the solver
// when outer solve is AztecOO as well.
solver = new AztecOO();
//solver.SetPrecOperator(precop_);
solver->SetAztecOption(AZ_solver, AZ_gmres);
// Do not use AZ_none
solver->SetAztecOption(AZ_precond, AZ_dom_decomp);
//solver->SetAztecOption(AZ_precond, AZ_none);
//solver->SetAztecOption(AZ_precond, AZ_Jacobi);
////solver->SetAztecOption(AZ_precond, AZ_Neumann);
//solver->SetAztecOption(AZ_overlap, 3);
//solver->SetAztecOption(AZ_subdomain_solve, AZ_ilu);
//solver->SetAztecOption(AZ_output, AZ_all);
//solver->SetAztecOption(AZ_diagnostics, AZ_all);
solver->SetProblem(Problem);
}
// What should be a good inner_tolerance :-) ?
solver->Iterate(config->inner_maxiters, config->inner_tolerance);
}
Epetra_MultiVector temp(BdMap, nvectors);
ssym->C->Multiply(false, Xs, temp);
temp.Update(1.0, Bd, -1.0);
//Epetra_SerialComm LComm; // Use Serial Comm for the local vectors.
//Epetra_Map LocalBdMap(-1, data->Dnr, data->DRowElems, 0, LComm);
//Epetra_MultiVector localrhs(LocalBdMap, nvectors);
//Epetra_MultiVector locallhs(LocalBdMap, nvectors);
//int lda;
//double *values;
err = temp.ExtractView(&values, &lda);
assert (err == 0);
//int nrows = data->Cptr->RowMap().NumMyElements();
// copy to local vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = localrhs.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
if (config->amesosForDiagonal)
{
ssym->LP->SetRHS(&localrhs);
ssym->LP->SetLHS(&locallhs);
ssym->Solver->Solve();
}
else
{
ssym->ifSolver->ApplyInverse(localrhs, locallhs);
}
err = locallhs.ExtractView(&values, &lda);
assert (err == 0);
// copy to distributed vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = temp.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
// For checking faults
//if (NumApplyInverse_ == 5) temp.ReplaceMyValue(0, 0, 0.0);
Epetra_Export XdExporter(BdMap, Y.Map());
Y.Export(temp, XdExporter, Insert);
Epetra_Export XsExporter(BsMap, Y.Map());
Y.Export(Xs, XsExporter, Insert);
if (config->libName == "Belos" || config->schurSolver == "Amesos")
{
// clean up
}
else
{
delete solver;
}
return 0;
}//end shylu_dist_solve <epetra,epetra>