// ======================================================================
void Amesos_Dscpack::PrintStatus() const
{
if (Problem_->GetOperator() != 0 && MyPID_ != 0)
{
std::string p = "Amesos_Dscpack : ";
PrintLine();
int n = GetProblem()->GetMatrix()->NumGlobalRows();
int nnz = GetProblem()->GetMatrix()->NumGlobalNonzeros();
std::cout << p << "Matrix has " << n << " rows"
<< " and " << nnz << " nonzeros" << std::endl;
std::cout << p << "Nonzero elements per row = "
<< 1.0 * nnz / n << std::endl;
std::cout << p << "Percentage of nonzero elements = "
<< 100.0 * nnz /(pow(n,2.0)) << std::endl;
std::cout << p << "Available process(es) = " << NumProcs_ << std::endl;
std::cout << p << "Process(es) used = " << DscNumProcs
<< ", idle = " << NumProcs_ - DscNumProcs << std::endl;
std::cout << p << "Estimated total memory for factorization = "
<< TotalMemory_ << " Mbytes" << std::endl;
}
if ( MyDscRank >= 0 )
DSC_DoStats( PrivateDscpackData_->MyDSCObject_ );
if (!MyPID_)
PrintLine();
return;
}
bool Amesos_Scalapack::MatrixShapeOK() const {
bool OK ;
if ( GetProblem()->GetOperator()->OperatorRangeMap().NumGlobalPoints() !=
GetProblem()->GetOperator()->OperatorDomainMap().NumGlobalPoints() ) OK = false;
return OK;
}
bool Amesos_Dscpack::MatrixShapeOK() const
{
bool OK = GetProblem()->IsOperatorSymmetric() ;
//
// The following test is redundant. I have left it here in case the
// IsOperatorSymmetric test turns out not to be reliable.
//
if ( GetProblem()->GetOperator()->OperatorRangeMap().NumGlobalPoints() !=
GetProblem()->GetOperator()->OperatorDomainMap().NumGlobalPoints() ) OK = false;
return OK;
}
//=============================================================================
bool Amesos_Klu::MatrixShapeOK() const {
bool OK = true;
// Comment by Tim: The following code seems suspect. The variable "OK"
// is not set if the condition is true.
// Does the variable "OK" default to true?
if ( GetProblem()->GetOperator()->OperatorRangeMap().NumGlobalPoints() !=
GetProblem()->GetOperator()->OperatorDomainMap().NumGlobalPoints() ) {
OK = false;
}
return OK;
}
//=============================================================================
int Amesos_Dscpack::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(0);
ResetTimer(1);
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
// MS // some checks on matrix size
if (RowMatrixA->NumGlobalRows() != RowMatrixA->NumGlobalCols())
AMESOS_CHK_ERR(-1);
// Convert vector b to a vector in the form that DSCPACK needs it
//
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong with input
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-2);
double *dscmapXvalues ;
int dscmapXlda ;
Epetra_MultiVector dscmapX(DscRowMap(),NumVectors) ;
int ierr;
AMESOS_CHK_ERR(dscmapX.ExtractView(&dscmapXvalues,&dscmapXlda));
assert (dscmapXlda == NumLocalCols);
double *dscmapBvalues ;
int dscmapBlda ;
Epetra_MultiVector dscmapB(DscRowMap(), NumVectors ) ;
ierr = dscmapB.ExtractView( &dscmapBvalues, &dscmapBlda );
AMESOS_CHK_ERR(ierr);
assert( dscmapBlda == NumLocalCols ) ;
AMESOS_CHK_ERR(dscmapB.Import(*vecB, Importer(), Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
ResetTimer(0);
// MS // now solve the problem
std::vector<double> ValuesInNewOrder( NumLocalCols ) ;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
if ( MyDscRank >= 0 ) {
for ( int j =0 ; j < NumVectors; j++ ) {
for ( int i = 0; i < NumLocalCols; i++ ) {
ValuesInNewOrder[i] = dscmapBvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapBlda ] ;
}
AMESOS_CHK_ERR( DSC_InputRhsLocalVec ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
AMESOS_CHK_ERR( DSC_Solve ( PrivateDscpackData_->MyDSCObject_ ) ) ;
AMESOS_CHK_ERR( DSC_GetLocalSolution ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
for ( int i = 0; i < NumLocalCols; i++ ) {
dscmapXvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapXlda ] = ValuesInNewOrder[i];
}
}
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
ResetTimer(0);
ResetTimer(1);
vecX->Export( dscmapX, Importer(), Insert ) ;
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
ComputeTrueResidual(*(GetProblem()->GetMatrix()), *vecX, *vecB,
false, "Amesos_Dscpack");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Dscpack");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
NumSolve_++;
return(0) ;
}
//=============================================================================
int Amesos_Dscpack::PerformSymbolicFactorization()
{
ResetTimer(0);
ResetTimer(1);
MyPID_ = Comm().MyPID();
NumProcs_ = Comm().NumProc();
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ;
const Epetra_MpiComm& comm1 = dynamic_cast<const Epetra_MpiComm &> (Comm());
int numrows = RowMatrixA->NumGlobalRows();
int numentries = RowMatrixA->NumGlobalNonzeros();
Teuchos::RCP<Epetra_CrsGraph> Graph;
Epetra_CrsMatrix* CastCrsMatrixA =
dynamic_cast<Epetra_CrsMatrix*>(RowMatrixA);
if (CastCrsMatrixA)
{
Graph = Teuchos::rcp(const_cast<Epetra_CrsGraph*>(&(CastCrsMatrixA->Graph())), false);
}
else
{
int MaxNumEntries = RowMatrixA->MaxNumEntries();
Graph = Teuchos::rcp(new Epetra_CrsGraph(Copy, OriginalMap, MaxNumEntries));
std::vector<int> Indices(MaxNumEntries);
std::vector<double> Values(MaxNumEntries);
for (int i = 0 ; i < RowMatrixA->NumMyRows() ; ++i)
{
int NumEntries;
RowMatrixA->ExtractMyRowCopy(i, MaxNumEntries, NumEntries,
&Values[0], &Indices[0]);
for (int j = 0 ; j < NumEntries ; ++j)
Indices[j] = RowMatrixA->RowMatrixColMap().GID(Indices[j]);
int GlobalRow = RowMatrixA->RowMatrixRowMap().GID(i);
Graph->InsertGlobalIndices(GlobalRow, NumEntries, &Indices[0]);
}
Graph->FillComplete();
}
//
// Create a replicated map and graph
//
std::vector<int> AllIDs( numrows ) ;
for ( int i = 0; i < numrows ; i++ ) AllIDs[i] = i ;
Epetra_Map ReplicatedMap( -1, numrows, &AllIDs[0], 0, Comm());
Epetra_Import ReplicatedImporter(ReplicatedMap, OriginalMap);
Epetra_CrsGraph ReplicatedGraph( Copy, ReplicatedMap, 0 );
AMESOS_CHK_ERR(ReplicatedGraph.Import(*Graph, ReplicatedImporter, Insert));
AMESOS_CHK_ERR(ReplicatedGraph.FillComplete());
//
// Convert the matrix to Ap, Ai
//
std::vector <int> Replicates(numrows);
std::vector <int> Ap(numrows + 1);
std::vector <int> Ai(EPETRA_MAX(numrows, numentries));
for( int i = 0 ; i < numrows; i++ ) Replicates[i] = 1;
int NumEntriesPerRow ;
int *ColIndices = 0 ;
int Ai_index = 0 ;
for ( int MyRow = 0; MyRow <numrows; MyRow++ ) {
AMESOS_CHK_ERR( ReplicatedGraph.ExtractMyRowView( MyRow, NumEntriesPerRow, ColIndices ) );
Ap[MyRow] = Ai_index ;
for ( int j = 0; j < NumEntriesPerRow; j++ ) {
Ai[Ai_index] = ColIndices[j] ;
Ai_index++;
}
}
assert( Ai_index == numentries ) ;
Ap[ numrows ] = Ai_index ;
MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0);
ResetTimer(0);
//
// Call Dscpack Symbolic Factorization
//
int OrderCode = 2;
std::vector<double> MyANonZ;
NumLocalNonz = 0 ;
GlobalStructNewColNum = 0 ;
GlobalStructNewNum = 0 ;
GlobalStructOwner = 0 ;
LocalStructOldNum = 0 ;
NumGlobalCols = 0 ;
// MS // Have to define the maximum number of processes to be used
// MS // This is only a suggestion as Dscpack uses a number of processes that is a power of 2
int NumGlobalNonzeros = GetProblem()->GetMatrix()->NumGlobalNonzeros();
int NumRows = GetProblem()->GetMatrix()->NumGlobalRows();
// optimal value for MaxProcs == -1
int OptNumProcs1 = 1+EPETRA_MAX( NumRows/10000, NumGlobalNonzeros/1000000 );
OptNumProcs1 = EPETRA_MIN(NumProcs_,OptNumProcs1 );
// optimal value for MaxProcs == -2
int OptNumProcs2 = (int)sqrt(1.0 * NumProcs_);
if( OptNumProcs2 < 1 ) OptNumProcs2 = 1;
// fix the value of MaxProcs
switch (MaxProcs_)
{
case -1:
MaxProcs_ = EPETRA_MIN(OptNumProcs1, NumProcs_);
break;
case -2:
MaxProcs_ = EPETRA_MIN(OptNumProcs2, NumProcs_);
break;
case -3:
MaxProcs_ = NumProcs_;
break;
}
#if 0
if (MyDscRank>=0 && A_and_LU_built) {
DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject);
}
#endif
// if ( ! A_and_LU_built ) {
// DSC_End( PrivateDscpackData_->MyDSCObject ) ;
// PrivateDscpackData_->MyDSCObject = DSC_Begin() ;
// }
// MS // here I continue with the old code...
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
DscNumProcs = 1 ;
int DscMax = DSC_Analyze( numrows, &Ap[0], &Ai[0], &Replicates[0] );
while ( DscNumProcs * 2 <=EPETRA_MIN( MaxProcs_, DscMax ) ) DscNumProcs *= 2 ;
MyDscRank = -1;
DSC_Open0( PrivateDscpackData_->MyDSCObject_, DscNumProcs, &MyDscRank, comm1.Comm()) ;
NumLocalCols = 0 ; // This is for those processes not in the Dsc grid
if ( MyDscRank >= 0 ) {
assert( MyPID_ == MyDscRank ) ;
AMESOS_CHK_ERR( DSC_Order ( PrivateDscpackData_->MyDSCObject_, OrderCode, numrows, &Ap[0], &Ai[0],
&Replicates[0], &NumGlobalCols, &NumLocalStructs,
&NumLocalCols, &NumLocalNonz,
&GlobalStructNewColNum, &GlobalStructNewNum,
&GlobalStructOwner, &LocalStructOldNum ) ) ;
assert( NumGlobalCols == numrows ) ;
assert( NumLocalCols == NumLocalStructs ) ;
}
if ( MyDscRank >= 0 ) {
int MaxSingleBlock;
const int Limit = 5000000 ; // Memory Limit set to 5 Terabytes
AMESOS_CHK_ERR( DSC_SFactor ( PrivateDscpackData_->MyDSCObject_, &TotalMemory_,
&MaxSingleBlock, Limit, DSC_LBLAS3, DSC_DBLAS2 ) ) ;
}
// A_and_LU_built = true; // If you uncomment this, TestOptions fails
SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_, 0);
return(0);
}
int Amesos_Klu::CreateLocalMatrixAndExporters()
{
ResetTimer(0);
RowMatrixA_ = Problem_->GetMatrix(); // MS, 18-Apr-06 //
if (RowMatrixA_ == 0) AMESOS_CHK_ERR(-1);
const Epetra_Map &OriginalMatrixMap = RowMatrixA_->RowMatrixRowMap() ;
const Epetra_Map &OriginalDomainMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorRangeMap():
GetProblem()->GetOperator()->OperatorDomainMap();
const Epetra_Map &OriginalRangeMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorDomainMap():
GetProblem()->GetOperator()->OperatorRangeMap();
NumGlobalElements_ = RowMatrixA_->NumGlobalRows();
numentries_ = RowMatrixA_->NumGlobalNonzeros();
int indexBase = OriginalMatrixMap.IndexBase();
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
//
// Create a serial matrix
//
assert( NumGlobalElements_ == OriginalMatrixMap.NumGlobalElements() ) ;
int NumMyElements_ = 0 ;
if (MyPID_==0) NumMyElements_ = NumGlobalElements_;
//
// UseDataInPlace_ is set to 1 (true) only if everything is perfectly
// normal. Anything out of the ordinary reverts to the more expensive
// path.
//
UseDataInPlace_ = ( OriginalMatrixMap.NumMyElements() ==
OriginalMatrixMap.NumGlobalElements() )?1:0;
if ( ! OriginalRangeMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( ! OriginalDomainMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( AddZeroToDiag_ ) UseDataInPlace_ = 0 ;
Comm().Broadcast( &UseDataInPlace_, 1, 0 ) ;
//
// Reindex matrix if necessary (and possible - i.e. CrsMatrix)
//
// For now, since I don't know how to determine if we need to reindex the matrix,
// we allow the user to control re-indexing through the "Reindex" parameter.
//
CrsMatrixA_ = dynamic_cast<Epetra_CrsMatrix *>(Problem_->GetOperator());
if ( Reindex_ ) {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR(-7);
#ifdef HAVE_AMESOS_EPETRAEXT
const Epetra_Map& OriginalMap = CrsMatrixA_->RowMap();
StdIndex_ = rcp( new Amesos_StandardIndex( OriginalMap ) );
//const Epetra_Map& OriginalColMap = CrsMatrixA_->RowMap();
StdIndexDomain_ = rcp( new Amesos_StandardIndex( OriginalDomainMap ) );
StdIndexRange_ = rcp( new Amesos_StandardIndex( OriginalRangeMap ) );
StdIndexMatrix_ = StdIndex_->StandardizeIndex( CrsMatrixA_ );
#else
std::cerr << "Amesos_Klu requires EpetraExt to reindex matrices." << std::endl
<< " Please rebuild with the EpetraExt library by adding --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR(-8);
#endif
} else {
StdIndexMatrix_ = RowMatrixA_ ;
}
//
// At this point, StdIndexMatrix_ points to a matrix with
// standard indexing.
//
//
// Convert Original Matrix to Serial (if it is not already)
//
if (UseDataInPlace_ == 1) {
SerialMatrix_ = StdIndexMatrix_;
} else {
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_, NumMyElements_, indexBase, Comm()));
if ( Problem_->GetRHS() )
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
else
NumVectors_ = 1 ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
ImportToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->RowMatrixRowMap() ) );
if (ImportToSerial_.get() == 0) AMESOS_CHK_ERR(-9);
// Build the vector data import/export once and only once
#define CHRIS
#ifdef CHRIS
if(StdIndexMatrix_->RowMatrixRowMap().SameAs(StdIndexMatrix_->OperatorRangeMap()))
ImportRangeToSerial_=ImportToSerial_;
else
ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->OperatorRangeMap() ) );
if(StdIndexMatrix_->RowMatrixRowMap().SameAs(StdIndexMatrix_->OperatorDomainMap()))
ImportDomainToSerial_=ImportToSerial_;
else
ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->OperatorDomainMap() ) );
#endif
SerialCrsMatrixA_ = rcp( new Epetra_CrsMatrix(Copy, *SerialMap_, 0) ) ;
SerialMatrix_ = &*SerialCrsMatrixA_ ;
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
return(0);
}
int Amesos_Paraklete::CreateLocalMatrixAndExporters()
{
ResetTimer(0);
RowMatrixA_ = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator());
if (RowMatrixA_ == 0) AMESOS_CHK_ERR(-1);
const Epetra_Map &OriginalMatrixMap = RowMatrixA_->RowMatrixRowMap() ;
const Epetra_Map &OriginalDomainMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorRangeMap():
GetProblem()->GetOperator()->OperatorDomainMap();
const Epetra_Map &OriginalRangeMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorDomainMap():
GetProblem()->GetOperator()->OperatorRangeMap();
NumGlobalElements_ = RowMatrixA_->NumGlobalRows();
numentries_ = RowMatrixA_->NumGlobalNonzeros();
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
//
// Create a serial matrix
//
assert( NumGlobalElements_ == OriginalMatrixMap.NumGlobalElements() ) ;
int NumMyElements_ = 0 ;
if (MyPID_==0) NumMyElements_ = NumGlobalElements_;
//
// UseDataInPlace_ is set to 1 (true) only if everything is perfectly
// normal. Anything out of the ordinary reverts to the more expensive
// path.
//
UseDataInPlace_ = ( OriginalMatrixMap.NumMyElements() ==
OriginalMatrixMap.NumGlobalElements() )?1:0;
if ( ! OriginalRangeMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( ! OriginalDomainMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( AddZeroToDiag_ ) UseDataInPlace_ = 0 ;
Comm().Broadcast( &UseDataInPlace_, 1, 0 ) ;
UseDataInPlace_ = 0 ; // bug - remove this someday.
//
// Reindex matrix if necessary (and possible - i.e. CrsMatrix)
//
// For now, since I don't know how to determine if we need to reindex the matrix,
// we allow the user to control re-indexing through the "Reindex" parameter.
//
CrsMatrixA_ = dynamic_cast<Epetra_CrsMatrix *>(Problem_->GetOperator());
Epetra_CrsMatrix* CcsMatrixA = 0 ;
//
// CcsMatrixA points to a matrix in Compressed Column Format
// i.e. the format needed by Paraklete. If we are solving
// A^T x = b, CcsMatrixA = CrsMatrixA_, otherwise we must
// transpose the matrix.
//
if (UseTranspose()) {
CcsMatrixA = CrsMatrixA_ ;
} else {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR( -7 ); // Amesos_Paraklete only supports CrsMatrix objects in the non-transpose case
#ifdef HAVE_AMESOS_EPETRAEXT
bool MakeDataContiguous = true;
transposer_ = rcp ( new EpetraExt::RowMatrix_Transpose( MakeDataContiguous ));
int OriginalTracebackMode = CrsMatrixA_->GetTracebackMode();
CrsMatrixA_->SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) );
CcsMatrixA = &(dynamic_cast<Epetra_CrsMatrix&>(((*transposer_)(*CrsMatrixA_))));
CrsMatrixA_->SetTracebackMode( OriginalTracebackMode );
#else
std::cerr << "Amesos_Paraklete requires the EpetraExt library to solve non-transposed problems. " << std::endl ;
std::cerr << " To rebuild Amesos with EpetraExt, add --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR( -3 );
#endif
}
if ( Reindex_ ) {
if ( CcsMatrixA == 0 ) AMESOS_CHK_ERR(-4);
#ifdef HAVE_AMESOS_EPETRAEXT
const Epetra_Map& OriginalMap = CcsMatrixA->RowMap();
StdIndex_ = rcp( new Amesos_StandardIndex( OriginalMap ) );
//const Epetra_Map& OriginalColMap = CcsMatrixA->RowMap();
StdIndexDomain_ = rcp( new Amesos_StandardIndex( OriginalDomainMap ) );
StdIndexRange_ = rcp( new Amesos_StandardIndex( OriginalRangeMap ) );
StdIndexMatrix_ = StdIndex_->StandardizeIndex( CcsMatrixA );
#else
std::cerr << "Amesos_Paraklete requires EpetraExt to reindex matrices." << std::endl
<< " Please rebuild with the EpetraExt library by adding --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR(-4);
#endif
} else {
if ( UseTranspose() ){
StdIndexMatrix_ = RowMatrixA_ ;
} else {
StdIndexMatrix_ = CcsMatrixA ;
}
}
//
// At this point, StdIndexMatrix_ points to a matrix with
// standard indexing.
//
//
// Convert Original Matrix to Serial (if it is not already)
//
if (UseDataInPlace_ == 1) {
SerialMatrix_ = StdIndexMatrix_;
} else {
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_, NumMyElements_, 0, Comm()));
if ( Problem_->GetRHS() )
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
else
NumVectors_ = 1 ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
ImportToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->RowMatrixRowMap() ) );
if (ImportToSerial_.get() == 0) AMESOS_CHK_ERR(-5);
SerialCrsMatrixA_ = rcp( new Epetra_CrsMatrix(Copy, *SerialMap_, 0) ) ;
SerialMatrix_ = &*SerialCrsMatrixA_ ;
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
return(0);
}