void show_matrix(const char *txt, const Epetra_LinearProblem &problem, const Epetra_Comm &comm)
{
int me = comm.MyPID();
if (comm.NumProc() > 10){
if (me == 0){
std::cout << txt << std::endl;
std::cout << "Printed matrix format only works for 10 or fewer processes" << std::endl;
}
return;
}
Epetra_RowMatrix *matrix = problem.GetMatrix();
Epetra_MultiVector *lhs = problem.GetLHS();
Epetra_MultiVector *rhs = problem.GetRHS();
int numRows = matrix->NumGlobalRows();
int numCols = matrix->NumGlobalCols();
if ((numRows > 200) || (numCols > 500)){
if (me == 0){
std::cerr << txt << std::endl;
std::cerr << "show_matrix: problem is too large to display" << std::endl;
}
return;
}
int *myA = new int [numRows * numCols];
make_my_A(*matrix, myA, comm);
int *myX = new int [numCols];
int *myB = new int [numRows];
memset(myX, 0, sizeof(int) * numCols);
memset(myB, 0, sizeof(int) * numRows);
const Epetra_BlockMap &lhsMap = lhs->Map();
const Epetra_BlockMap &rhsMap = rhs->Map();
int base = lhsMap.IndexBase();
for (int j=0; j < lhsMap.NumMyElements(); j++){
int colGID = lhsMap.GID(j);
myX[colGID - base] = me + 1;
}
for (int i=0; i < rhsMap.NumMyElements(); i++){
int rowGID = rhsMap.GID(i);
myB[rowGID - base] = me + 1;
}
printMatrix(txt, myA, myX, myB, numRows, numCols, comm);
delete [] myA;
delete [] myX;
delete [] myB;
}
// ======================================================================
bool BasicTest(string PrecType,
CrsMatrixGallery& Gallery)
{
// The following methods of CrsMatrixGallery are used to get pointers
// to internally stored Epetra_RowMatrix and Epetra_LinearProblem.
Epetra_RowMatrix* A = Gallery.GetMatrix();
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
Epetra_MultiVector& RHS = *(Problem->GetRHS());
Epetra_MultiVector& LHS = *(Problem->GetLHS());
LHS.PutScalar(0.0);
// Set up the list
Teuchos::ParameterList List;
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: sweeps",1550);
List.set("relaxation: type", PrecType);
Ifpack_PointRelaxation Point(A);
Point.SetParameters(List);
Point.Compute();
// use the preconditioner as solver, with 1550 iterations
Point.ApplyInverse(RHS,LHS);
// compute the real residual
double residual, diff;
Gallery.ComputeResidual(&residual);
Gallery.ComputeDiffBetweenStartingAndExactSolutions(&diff);
if (verbose && A->Comm().MyPID()==0) {
cout << "||b-Ax||_2 = " << residual << endl;
cout << "||x_exact - x||_2 = " << diff << endl;
}
// Jacobi is very slow to converge here
if (residual < 1e-2) {
if (verbose)
cout << "Test passed" << endl;
return(true);
}
else {
if (verbose)
cout << "Test failed!" << endl;
return(false);
}
}
// ======================================================================
int CompareBlockSizes(string PrecType,
CrsMatrixGallery& Gallery, int NumParts)
{
Epetra_RowMatrix* A = Gallery.GetMatrix();
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
Epetra_MultiVector& RHS = *(Problem->GetRHS());
Epetra_MultiVector& LHS = *(Problem->GetLHS());
Teuchos::ParameterList List;
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: type", PrecType);
List.set("relaxation: sweeps",1);
List.set("partitioner: type", "greedy");
List.set("partitioner: local parts", NumParts);
RHS.PutScalar(1.0);
LHS.PutScalar(0.0);
Ifpack_AdditiveSchwarz<Ifpack_BlockRelaxation<Ifpack_DenseContainer> > Prec(A);
Prec.SetParameters(List);
Prec.Compute();
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
if (verbose)
AztecOOSolver.SetAztecOption(AZ_output,32);
else
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Prec);
AztecOOSolver.Iterate(1550,1e-8);
return(AztecOOSolver.NumIters());
}
int TestMultiLevelPreconditioner(char ProblemType[],
Teuchos::ParameterList & MLList,
Epetra_LinearProblem & Problem, double & TotalErrorResidual,
double & TotalErrorExactSol,bool cg=false)
{
Epetra_MultiVector* lhs = Problem.GetLHS();
Epetra_MultiVector* rhs = Problem.GetRHS();
Epetra_RowMatrix* A = Problem.GetMatrix();
// ======================================== //
// create a rhs corresponding to lhs or 1's //
// ======================================== //
lhs->PutScalar(1.0);
A->Multiply(false,*lhs,*rhs);
lhs->PutScalar(0.0);
Epetra_Time Time(A->Comm());
// =================== //
// call ML and AztecOO //
// =================== //
AztecOO solver(Problem);
MLList.set("ML output", 10);
ML_Epetra::MultiLevelPreconditioner * MLPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
// tell AztecOO to use this preconditioner, then solve
solver.SetPrecOperator(MLPrec);
if(cg) solver.SetAztecOption(AZ_solver, AZ_cg);
else solver.SetAztecOption(AZ_solver, AZ_gmres);
solver.SetAztecOption(AZ_output, 32);
solver.SetAztecOption(AZ_kspace, 160);
solver.Iterate(1550, 1e-12);
delete MLPrec;
// ==================================================== //
// compute difference between exact solution and ML one //
// ==================================================== //
double d = 0.0, d_tot = 0.0;
for( int i=0 ; i<lhs->Map().NumMyElements() ; ++i )
d += ((*lhs)[0][i] - 1.0) * ((*lhs)[0][i] - 1.0);
A->Comm().SumAll(&d,&d_tot,1);
// ================== //
// compute ||Ax - b|| //
// ================== //
double Norm;
Epetra_Vector Ax(rhs->Map());
A->Multiply(false, *lhs, Ax);
Ax.Update(1.0, *rhs, -1.0);
Ax.Norm2(&Norm);
string msg = ProblemType;
if (A->Comm().MyPID() == 0) {
cout << msg << "......Using " << A->Comm().NumProc() << " processes" << endl;
cout << msg << "......||A x - b||_2 = " << Norm << endl;
cout << msg << "......||x_exact - x||_2 = " << sqrt(d_tot) << endl;
cout << msg << "......Total Time = " << Time.ElapsedTime() << endl;
}
TotalErrorExactSol += sqrt(d_tot);
TotalErrorResidual += Norm;
return( solver.NumIters() );
}
int TestMultiLevelPreconditioner(char ProblemType[],
Teuchos::ParameterList & MLList,
Epetra_LinearProblem & Problem, double & TotalErrorResidual,
double & TotalErrorExactSol)
{
Epetra_MultiVector* lhs = Problem.GetLHS();
Epetra_MultiVector* rhs = Problem.GetRHS();
Epetra_CrsMatrix* A = dynamic_cast<Epetra_CrsMatrix*>(Problem.GetMatrix());
int PID = A->Comm().MyPID();
int numProcs = A->Comm().NumProc();
RCP<const Epetra_RowMatrix> Arcp = Teuchos::rcp(A, false);
double n1, n2,nf;
// ======================================== //
// create a rhs corresponding to lhs or 1's //
// ======================================== //
lhs->PutScalar(1.0);
A->Multiply(false,*lhs,*rhs);
lhs->PutScalar(0.0);
MLList.set("ML output", 0);
RowMatrixToMatlabFile("mat_f.dat",*A);
MultiVectorToMatrixMarketFile("lhs_f.dat",*lhs,0,0,false);
MultiVectorToMatrixMarketFile("rhs_f.dat",*rhs,0,0,false);
Epetra_Time Time(A->Comm());
/* Build the Zoltan list - Group #1 */
ParameterList Zlist1,Sublist1;
Sublist1.set("DEBUG_LEVEL","0");
Sublist1.set("NUM_GLOBAL_PARTITIONS","2");
Zlist1.set("Zoltan",Sublist1);
/* Start Isorropia's Ninja Magic - Group #1 */
RefCountPtr<Isorropia::Epetra::Partitioner> partitioner1 =
Isorropia::Epetra::create_partitioner(Arcp, Zlist1);
Isorropia::Epetra::Redistributor rd1(partitioner1);
Teuchos::RCP<Epetra_CrsMatrix> ResA1=rd1.redistribute(*A);
Teuchos::RCP<Epetra_MultiVector> ResX1=rd1.redistribute(*lhs);
Teuchos::RCP<Epetra_MultiVector> ResB1=rd1.redistribute(*rhs);
RestrictedCrsMatrixWrapper RW1;
RW1.restrict_comm(ResA1);
RestrictedMultiVectorWrapper RX1,RB1;
RX1.restrict_comm(ResX1);
RB1.restrict_comm(ResB1);
/* Build the Zoltan list - Group #2 */
ParameterList Zlist2,Sublist2;
Sublist2.set("DEBUG_LEVEL","0");
if(PID > 1) Sublist2.set("NUM_LOCAL_PARTITIONS","1");
else Sublist2.set("NUM_LOCAL_PARTITIONS","0");
Zlist2.set("Zoltan",Sublist2);
/* Start Isorropia's Ninja Magic - Group #2 */
RefCountPtr<Isorropia::Epetra::Partitioner> partitioner2 =
Isorropia::Epetra::create_partitioner(Arcp, Zlist2);
Isorropia::Epetra::Redistributor rd2(partitioner2);
Teuchos::RCP<Epetra_CrsMatrix> ResA2=rd2.redistribute(*A);
Teuchos::RCP<Epetra_MultiVector> ResX2=rd2.redistribute(*lhs);
Teuchos::RCP<Epetra_MultiVector> ResB2=rd2.redistribute(*rhs);
RestrictedCrsMatrixWrapper RW2;
RW2.restrict_comm(ResA2);
RestrictedMultiVectorWrapper RX2,RB2;
RX2.restrict_comm(ResX2);
RB2.restrict_comm(ResB2);
if(RW1.RestrictedProcIsActive()){
Teuchos::RCP<Epetra_CrsMatrix> SubA1 = RW1.RestrictedMatrix();
Teuchos::RCP<Epetra_MultiVector> SubX1 = RX1.RestrictedMultiVector();
Teuchos::RCP<Epetra_MultiVector> SubB1 = RB1.RestrictedMultiVector();
ML_Epetra::MultiLevelPreconditioner * SubPrec1 = new ML_Epetra::MultiLevelPreconditioner(*SubA1, MLList, true);
Epetra_LinearProblem Problem1(&*SubA1,&*SubX1,&*SubB1);
AztecOO solver1(Problem1);
solver1.SetPrecOperator(SubPrec1);
solver1.SetAztecOption(AZ_solver, AZ_gmres);
solver1.SetAztecOption(AZ_output, 32);
solver1.SetAztecOption(AZ_kspace, 160);
solver1.Iterate(1550, 1e-12);
delete SubPrec1;
}
else{
Teuchos::RCP<Epetra_CrsMatrix> SubA2 = RW2.RestrictedMatrix();
Teuchos::RCP<Epetra_MultiVector> SubX2 = RX2.RestrictedMultiVector();
Teuchos::RCP<Epetra_MultiVector> SubB2 = RB2.RestrictedMultiVector();
ML_Epetra::MultiLevelPreconditioner * SubPrec2 = new ML_Epetra::MultiLevelPreconditioner(*SubA2, MLList, true);
Epetra_LinearProblem Problem2(&*SubA2,&*SubX2,&*SubB2);
AztecOO solver2(Problem2);
solver2.SetPrecOperator(SubPrec2);
solver2.SetAztecOption(AZ_solver, AZ_gmres);
solver2.SetAztecOption(AZ_output, 32);
solver2.SetAztecOption(AZ_kspace, 160);
solver2.Iterate(1550, 1e-12);
delete SubPrec2;
}
/* Post-processing exports */
Epetra_MultiVector ans1(*lhs), ans2(*lhs);
rd1.redistribute_reverse(*ResX1,ans1);
rd2.redistribute_reverse(*ResX2,ans2);
/* Run on Full Problem */
A->Comm().Barrier();
ML_Epetra::MultiLevelPreconditioner * FullPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
AztecOO solverF(Problem);
solverF.SetPrecOperator(FullPrec);
solverF.SetAztecOption(AZ_solver, AZ_gmres);
solverF.SetAztecOption(AZ_output, 32);
solverF.SetAztecOption(AZ_kspace, 160);
solverF.Iterate(1550, 1e-12);
delete FullPrec;
/* Solution Comparison */
ans1.Update(1.0,*lhs,-1.0);
ans2.Update(1.0,*lhs,-1.0);
ans1.Norm2(&n1);
ans2.Norm2(&n2);
if(!PID) {
printf("Norm Diff 1 = %6.4e\n",n1);
printf("Norm Diff 2 = %6.4e\n",n2);
}
TotalErrorExactSol += n1 + n2;
}
// ======================================================================
bool ComparePointAndBlock(string PrecType,
CrsMatrixGallery& Gallery, int sweeps)
{
Epetra_RowMatrix* A = Gallery.GetMatrix();
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
Epetra_MultiVector& RHS = *(Problem->GetRHS());
Epetra_MultiVector& LHS = *(Problem->GetLHS());
// Set up the list
Teuchos::ParameterList List;
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: type", PrecType);
List.set("relaxation: sweeps",sweeps);
List.set("partitioner: type", "linear");
List.set("partitioner: local parts", A->NumMyRows());
int ItersPoint, ItersBlock;
// ================================================== //
// get the number of iterations with point relaxation //
// ================================================== //
{
RHS.PutScalar(1.0);
LHS.PutScalar(0.0);
Ifpack_AdditiveSchwarz<Ifpack_PointRelaxation> Point(A);
Point.SetParameters(List);
Point.Compute();
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
if (verbose)
AztecOOSolver.SetAztecOption(AZ_output,32);
else
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Point);
AztecOOSolver.Iterate(1550,1e-8);
double TrueResidual = AztecOOSolver.TrueResidual();
ItersPoint = AztecOOSolver.NumIters();
// some output
if (verbose && Problem->GetMatrix()->Comm().MyPID() == 0) {
cout << "Iterations = " << ItersPoint << endl;
cout << "Norm of the true residual = " << TrueResidual << endl;
}
}
// ================================================== //
// get the number of iterations with block relaxation //
// ================================================== //
{
RHS.PutScalar(1.0);
LHS.PutScalar(0.0);
Ifpack_AdditiveSchwarz<Ifpack_BlockRelaxation<Ifpack_DenseContainer> > Block(A);
Block.SetParameters(List);
Block.Compute();
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
if (verbose)
AztecOOSolver.SetAztecOption(AZ_output,32);
else
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Block);
AztecOOSolver.Iterate(1550,1e-8);
double TrueResidual = AztecOOSolver.TrueResidual();
ItersBlock = AztecOOSolver.NumIters();
// some output
if (verbose && Problem->GetMatrix()->Comm().MyPID() == 0) {
cout << "Iterations " << ItersBlock << endl;
cout << "Norm of the true residual = " << TrueResidual << endl;
}
}
if (ItersPoint != ItersBlock) {
if (verbose)
cout << "TEST FAILED!" << endl;
return(false);
}
else {
if (verbose)
cout << "TEST PASSED" << endl;
return(true);
}
}
bool CompareWithAztecOO(Epetra_LinearProblem& Problem, const std::string what,
int Overlap, int ival)
{
using std::cout;
using std::endl;
AztecOO AztecOOSolver(Problem);
AztecOOSolver.SetAztecOption(AZ_solver,AZ_gmres);
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetAztecOption(AZ_overlap,Overlap);
AztecOOSolver.SetAztecOption(AZ_graph_fill,ival);
AztecOOSolver.SetAztecOption(AZ_poly_ord, ival);
AztecOOSolver.SetAztecParam(AZ_drop, 0.0);
AztecOOSolver.SetAztecParam(AZ_athresh, 0.0);
AztecOOSolver.SetAztecParam(AZ_rthresh, 0.0);
Epetra_MultiVector& RHS = *(Problem.GetRHS());
Epetra_MultiVector& LHS = *(Problem.GetLHS());
Teuchos::RefCountPtr<Epetra_RowMatrix> A = Teuchos::rcp(Problem.GetMatrix(), false);
LHS.Random();
A->Multiply(false,LHS,RHS);
Teuchos::ParameterList List;
List.set("fact: level-of-fill", ival);
List.set("relaxation: sweeps", ival);
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: zero starting solution", true);
//default combine mode is as for AztecOO
List.set("schwarz: combine mode", Zero);
Epetra_Time Time(A->Comm());
Teuchos::RefCountPtr<Ifpack_Preconditioner> Prec;
if (what == "Jacobi") {
Prec = Teuchos::rcp( new Ifpack_PointRelaxation(&*A) );
List.set("relaxation: type", "Jacobi");
AztecOOSolver.SetAztecOption(AZ_precond,AZ_Jacobi);
AztecOOSolver.SetAztecOption(AZ_reorder,0);
}
else if (what == "IC no reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_IC>(&*A,Overlap) );
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_icc);
AztecOOSolver.SetAztecOption(AZ_reorder,0);
}
else if (what == "IC reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_IC>(&*A,Overlap) );
List.set("schwarz: use reordering", true);
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_icc);
AztecOOSolver.SetAztecOption(AZ_reorder,1);
}
else if (what == "ILU no reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_ILU>(&*A,Overlap) );
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_ilu);
AztecOOSolver.SetAztecOption(AZ_reorder,0);
}
else if (what == "ILU reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_ILU>(&*A,Overlap) );
List.set("schwarz: use reordering", true);
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_ilu);
AztecOOSolver.SetAztecOption(AZ_reorder,1);
}
#ifdef HAVE_IFPACK_AMESOS
else if (what == "LU") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_Amesos>(&*A,Overlap) );
List.set("amesos: solver type", "Klu");
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_lu);
}
#endif
else {
cerr << "Option not recognized" << endl;
exit(EXIT_FAILURE);
}
// ==================================== //
// Solve with AztecOO's preconditioners //
// ==================================== //
LHS.PutScalar(0.0);
Time.ResetStartTime();
AztecOOSolver.Iterate(150,1e-5);
if (verbose) {
cout << endl;
cout << "==================================================" << endl;
cout << "Testing `" << what << "', Overlap = "
<< Overlap << ", ival = " << ival << endl;
cout << endl;
cout << "[AztecOO] Total time = " << Time.ElapsedTime() << " (s)" << endl;
cout << "[AztecOO] Residual = " << AztecOOSolver.TrueResidual() << " (s)" << endl;
cout << "[AztecOO] Iterations = " << AztecOOSolver.NumIters() << endl;
cout << endl;
}
int AztecOOPrecIters = AztecOOSolver.NumIters();
// =========================================== //
// Create the IFPACK preconditioner and solver //
// =========================================== //
Epetra_Time Time2(A->Comm());
assert(Prec != Teuchos::null);
IFPACK_CHK_ERR(Prec->SetParameters(List));
Time.ResetStartTime();
IFPACK_CHK_ERR(Prec->Initialize());
if (verbose)
cout << "[IFPACK] Time for Initialize() = "
<< Time.ElapsedTime() << " (s)" << endl;
Time.ResetStartTime();
IFPACK_CHK_ERR(Prec->Compute());
if (verbose)
cout << "[IFPACK] Time for Compute() = "
<< Time.ElapsedTime() << " (s)" << endl;
AztecOOSolver.SetPrecOperator(&*Prec);
LHS.PutScalar(0.0);
Time.ResetStartTime();
AztecOOSolver.Iterate(150,1e-5);
if (verbose) {
cout << "[IFPACK] Total time = " << Time2.ElapsedTime() << " (s)" << endl;
cout << "[IFPACK] Residual = " << AztecOOSolver.TrueResidual() << " (s)" << endl;
cout << "[IFPACK] Iterations = " << AztecOOSolver.NumIters() << endl;
cout << endl;
}
int IFPACKPrecIters = AztecOOSolver.NumIters();
if (IFPACK_ABS(AztecOOPrecIters - IFPACKPrecIters) > 3) {
cerr << "TEST FAILED (" << AztecOOPrecIters << " != "
<< IFPACKPrecIters << ")" << endl;
return(false);
}
else
return(true);
}
// ============================================================================
void
Solve(const Epetra_LinearProblem Problem)
{
Solve(Problem.GetMatrix(), Problem.GetLHS(), Problem.GetRHS());
}
int main(int argc, char *argv[]) {
int returnierr=0;
using std::cout;
using std::endl;
using std::flush;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
int size; // Number of MPI processes, My process ID
MPI_Comm_size(MPI_COMM_WORLD, &size);
if (size > 1) {
cout << "This example cannot be run on more than one processor!" << endl;
MPI_Finalize();
returnierr = -1;
return returnierr;
}
#endif
bool verbose = false;
// Check if we should print results to standard out
if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true;
#ifdef EPETRA_MPI
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
if (!verbose) Comm.SetTracebackMode(0); // This should shut down any error traceback reporting
if (verbose) {
cout << EpetraExt::EpetraExt_Version() << endl << endl;
cout << Comm << endl << flush;
}
Comm.Barrier();
int NumMyElements = 3;
Epetra_Map Map( NumMyElements, 0, Comm );
Epetra_CrsGraph Graph( Copy, Map, 1 );
int index[2];
index[0] = 2;
Graph.InsertGlobalIndices( 0, 1, &index[0] );
index[0] = 0;
index[1] = 2;
Graph.InsertGlobalIndices( 1, 2, &index[0] );
index[0] = 1;
Graph.InsertGlobalIndices( 2, 1, &index[0] );
Graph.FillComplete();
if (verbose) {
cout << "***************** PERFORMING BTF TRANSFORM ON CRS_GRAPH *****************" <<endl<<endl;
cout << "CrsGraph *before* BTF transform: " << endl << endl;
cout << Graph << endl;
}
EpetraExt::AmesosBTF_CrsGraph BTFTrans( true, verbose );
Epetra_CrsGraph & NewBTFGraph = BTFTrans( Graph );
if (verbose) {
cout << "CrsGraph *after* BTF transform: " << endl << endl;
cout << NewBTFGraph << endl;
}
// Use BTF graph transformation to solve linear system.
// Create an Epetra::CrsMatrix
Epetra_CrsMatrix Matrix( Copy, Graph );
double value[2];
index[0] = 2; value[0] = 3.0;
Matrix.ReplaceMyValues( 0, 1, &value[0], &index[0] );
index[0] = 0; index[1] = 2;
value[0] = 2.0; value[1] = 2.5;
Matrix.ReplaceMyValues( 1, 2, &value[0], &index[0] );
index[0] = 1; value[0] = 1.0;
Matrix.ReplaceMyValues( 2, 1, &value[0], &index[0] );
Matrix.FillComplete();
// Create the solution and right-hand side vectors.
Epetra_MultiVector RHS( Map, 1 ), LHS( Map, 1 );
LHS.PutScalar( 0.0 );
RHS.ReplaceMyValue( 0, 0, 3.0 );
RHS.ReplaceMyValue( 1, 0, 4.5 );
RHS.ReplaceMyValue( 2, 0, 1.0 );
Epetra_LinearProblem problem( &Matrix, &LHS, &RHS );
if (verbose) {
cout << "*************** PERFORMING BTF TRANSFORM ON LINEAR_PROBLEM **************" <<endl<<endl;
cout << "CrsMatrix *before* BTF transform: " << endl << endl;
cout << Matrix << endl;
cout << "MultiVector RHS *before* BTF transform: " << endl << endl;
RHS.Print( cout );
}
// Create the linear problem transform.
EpetraExt::LinearProblem_GraphTrans * LPTrans =
new EpetraExt::LinearProblem_GraphTrans(
*(dynamic_cast<EpetraExt::StructuralSameTypeTransform<Epetra_CrsGraph>*>(&BTFTrans)) );
Epetra_LinearProblem* tProblem = &((*LPTrans)( problem ));
LPTrans->fwd();
if (verbose) {
cout << "CrsMatrix *after* BTF transform: " << endl << endl;
dynamic_cast<Epetra_CrsMatrix*>(tProblem->GetMatrix())->Print( cout );
cout << "MultiVector RHS *after* BTF transform: " << endl << endl;
tProblem->GetRHS()->Print( cout );
}
if (verbose) {
cout << endl << "*************** PERFORMING REINDEXING ON LINEAR_PROBLEM **************" <<endl<<endl;
}
EpetraExt::ViewTransform<Epetra_LinearProblem> * ReIdx_LPTrans =
new EpetraExt::LinearProblem_Reindex( 0 );
Epetra_LinearProblem* tProblem2 = &((*ReIdx_LPTrans)( *tProblem ));
ReIdx_LPTrans->fwd();
if (verbose) {
cout << endl << "CrsMatrix *after* BTF transform *and* reindexing: " << endl << endl;
dynamic_cast<Epetra_CrsMatrix*>(tProblem2->GetMatrix())->Print( cout );
cout << endl <<"Column Map *before* reindexing: " << endl << endl;
cout << dynamic_cast<Epetra_CrsMatrix*>(tProblem->GetMatrix())->ColMap() << endl;
cout << "Column Map *after* reindexing: " << endl << endl;
cout << dynamic_cast<Epetra_CrsMatrix*>(tProblem2->GetMatrix())->ColMap() << endl;
}
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return returnierr;
}
int main(int argc, char *argv[]) {
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
cout << Comm << endl;
int MyPID = Comm.MyPID();
bool verbose = false;
bool verbose1 = true;
if (MyPID==0) verbose = true;
if(argc < 2 && verbose) {
cerr << "Usage: " << argv[0]
<< " HB_filename [level_fill [level_overlap [absolute_threshold [ relative_threshold]]]]" << endl
<< "where:" << endl
<< "HB_filename - filename and path of a Harwell-Boeing data set" << endl
<< "level_fill - The amount of fill to use for ILU(k) preconditioner (default 0)" << endl
<< "level_overlap - The amount of overlap used for overlapping Schwarz subdomains (default 0)" << endl
<< "absolute_threshold - The minimum value to place on the diagonal prior to factorization (default 0.0)" << endl
<< "relative_threshold - The relative amount to perturb the diagonal prior to factorization (default 1.0)" << endl << endl
<< "To specify a non-default value for one of these parameters, you must specify all" << endl
<< " preceding values but not any subsequent parameters. Example:" << endl
<< "ifpackHpcSerialMsr.exe mymatrix.hpc 1 - loads mymatrix.hpc, uses level fill of one, all other values are defaults" << endl
<< endl;
return(1);
}
// Uncomment the next three lines to debug in mpi mode
//int tmp;
//if (MyPID==0) cin >> tmp;
//Comm.Barrier();
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra(argv[1], Comm, readMap, readA, readx, readb, readxexact);
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
// Create Exporter to distribute read-in matrix and vectors
Epetra_Export exporter(*readMap, map);
Epetra_CrsMatrix A(Copy, map, 0);
Epetra_Vector x(map);
Epetra_Vector b(map);
Epetra_Vector xexact(map);
Epetra_Time FillTimer(Comm);
x.Export(*readx, exporter, Add);
b.Export(*readb, exporter, Add);
xexact.Export(*readxexact, exporter, Add);
Comm.Barrier();
double vectorRedistributeTime = FillTimer.ElapsedTime();
A.Export(*readA, exporter, Add);
Comm.Barrier();
double matrixRedistributeTime = FillTimer.ElapsedTime() - vectorRedistributeTime;
assert(A.FillComplete()==0);
Comm.Barrier();
double fillCompleteTime = FillTimer.ElapsedTime() - matrixRedistributeTime;
if (Comm.MyPID()==0) {
cout << "\n\n****************************************************" << endl;
cout << "\n Vector redistribute time (sec) = " << vectorRedistributeTime<< endl;
cout << " Matrix redistribute time (sec) = " << matrixRedistributeTime << endl;
cout << " Transform to Local time (sec) = " << fillCompleteTime << endl<< endl;
}
Epetra_Vector tmp1(*readMap);
Epetra_Vector tmp2(map);
readA->Multiply(false, *readxexact, tmp1);
A.Multiply(false, xexact, tmp2);
double residual;
tmp1.Norm2(&residual);
if (verbose) cout << "Norm of Ax from file = " << residual << endl;
tmp2.Norm2(&residual);
if (verbose) cout << "Norm of Ax after redistribution = " << residual << endl << endl << endl;
//cout << "A from file = " << *readA << endl << endl << endl;
//cout << "A after dist = " << A << endl << endl << endl;
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
Comm.Barrier();
bool smallProblem = false;
if (A.RowMap().NumGlobalElements()<100) smallProblem = true;
if (smallProblem)
cout << "Original Matrix = " << endl << A << endl;
x.PutScalar(0.0);
Epetra_LinearProblem FullProblem(&A, &x, &b);
double normb, norma;
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
Epetra_Time ReductionTimer(Comm);
Epetra_CrsSingletonFilter SingletonFilter;
Comm.Barrier();
double reduceInitTime = ReductionTimer.ElapsedTime();
SingletonFilter.Analyze(&A);
Comm.Barrier();
double reduceAnalyzeTime = ReductionTimer.ElapsedTime() - reduceInitTime;
if (SingletonFilter.SingletonsDetected())
cout << "Singletons found" << endl;
else {
cout << "Singletons not found" << endl;
exit(1);
}
SingletonFilter.ConstructReducedProblem(&FullProblem);
Comm.Barrier();
double reduceConstructTime = ReductionTimer.ElapsedTime() - reduceInitTime;
double totalReduceTime = ReductionTimer.ElapsedTime();
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Reduction init time (sec) = " << reduceInitTime<< endl
<< " Reduction Analyze time (sec) = " << reduceAnalyzeTime << endl
<< " Construct Reduced Problem time (sec) = " << reduceConstructTime << endl
<< " Reduction Total time (sec) = " << totalReduceTime << endl<< endl;
Statistics(SingletonFilter);
Epetra_LinearProblem * ReducedProblem = SingletonFilter.ReducedProblem();
Epetra_CrsMatrix * Ap = dynamic_cast<Epetra_CrsMatrix *>(ReducedProblem->GetMatrix());
Epetra_Vector * bp = (*ReducedProblem->GetRHS())(0);
Epetra_Vector * xp = (*ReducedProblem->GetLHS())(0);
if (smallProblem)
cout << " Reduced Matrix = " << endl << *Ap << endl
<< " LHS before sol = " << endl << *xp << endl
<< " RHS = " << endl << *bp << endl;
// Construct ILU preconditioner
double elapsed_time, total_flops, MFLOPs;
Epetra_Time timer(Comm);
int LevelFill = 0;
if (argc > 2) LevelFill = atoi(argv[2]);
if (verbose) cout << "Using Level Fill = " << LevelFill << endl;
int Overlap = 0;
if (argc > 3) Overlap = atoi(argv[3]);
if (verbose) cout << "Using Level Overlap = " << Overlap << endl;
double Athresh = 0.0;
if (argc > 4) Athresh = atof(argv[4]);
if (verbose) cout << "Using Absolute Threshold Value of = " << Athresh << endl;
double Rthresh = 1.0;
if (argc > 5) Rthresh = atof(argv[5]);
if (verbose) cout << "Using Relative Threshold Value of = " << Rthresh << endl;
Ifpack_IlukGraph * IlukGraph = 0;
Ifpack_CrsRiluk * ILUK = 0;
if (LevelFill>-1) {
elapsed_time = timer.ElapsedTime();
IlukGraph = new Ifpack_IlukGraph(Ap->Graph(), LevelFill, Overlap);
assert(IlukGraph->ConstructFilledGraph()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
if (verbose) cout << "Time to construct ILUK graph = " << elapsed_time << endl;
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
ILUK = new Ifpack_CrsRiluk(*IlukGraph);
ILUK->SetFlopCounter(fact_counter);
ILUK->SetAbsoluteThreshold(Athresh);
ILUK->SetRelativeThreshold(Rthresh);
//assert(ILUK->InitValues()==0);
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
//cout << *ILUK << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
int Maxiter = 100;
double Tolerance = 1.0E-8;
Epetra_Flops counter;
Ap->SetFlopCounter(counter);
xp->SetFlopCounter(*Ap);
bp->SetFlopCounter(*Ap);
if (ILUK!=0) ILUK->SetFlopCounter(*Ap);
elapsed_time = timer.ElapsedTime();
double normreducedb, normreduceda;
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
Epetra_Vector resid(x);
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
double normx, normxexact;
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (verbose)
cout << "********************************************************" << endl
<< " Solving again with 2*Ax=2*b" << endl
<< "********************************************************" << endl;
A.Scale(1.0); // A = 2*A
b.Scale(1.0); // b = 2*b
x.PutScalar(0.0);
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
double updateReducedProblemTime = ReductionTimer.ElapsedTime();
SingletonFilter.UpdateReducedProblem(&FullProblem);
Comm.Barrier();
updateReducedProblemTime = ReductionTimer.ElapsedTime() - updateReducedProblemTime;
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Update Reduced Problem time (sec) = " << updateReducedProblemTime<< endl
<< "****************************************************" << endl;
Statistics(SingletonFilter);
if (LevelFill>-1) {
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (ILUK!=0) delete ILUK;
if (IlukGraph!=0) delete IlukGraph;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return 0 ;
}
int TestMultiLevelPreconditioner(char ProblemType[],
Teuchos::ParameterList & MLList,
Epetra_LinearProblem & Problem, double & TotalErrorResidual,
double & TotalErrorExactSol)
{
Epetra_MultiVector* lhs = Problem.GetLHS();
Epetra_MultiVector* rhs = Problem.GetRHS();
Epetra_RowMatrix* A = Problem.GetMatrix();
// ======================================== //
// create a rhs corresponding to lhs or 1's //
// ======================================== //
lhs->PutScalar(1.0);
A->Multiply(false,*lhs,*rhs);
lhs->PutScalar(0.0);
Epetra_Time Time(A->Comm());
Epetra_MultiVector lhs2(*lhs);
Epetra_MultiVector rhs2(*rhs);
// =================== //
// call ML and AztecOO //
// =================== //
AztecOO solver(Problem);
MLList.set("ML output", 0);
ML_set_random_seed(24601);
ML_Epetra::MultiLevelPreconditioner * MLPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
// tell AztecOO to use this preconditioner, then solve
solver.SetPrecOperator(MLPrec);
solver.SetAztecOption(AZ_solver, AZ_gmres);
solver.SetAztecOption(AZ_output, 32);
solver.SetAztecOption(AZ_kspace, 160);
solver.Iterate(1550, 1e-12);
delete MLPrec;
// ================================= //
// call ML and AztecOO a second time //
// ================================= //
Epetra_LinearProblem Problem2(A,&lhs2,&rhs2);
AztecOO solver2(Problem2);
ML_set_random_seed(24601);
ML_Epetra::MultiLevelPreconditioner * MLPrec2 = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
// tell AztecOO to use this preconditioner, then solve
solver2.SetPrecOperator(MLPrec2);
solver2.SetAztecOption(AZ_solver, AZ_gmres);
solver2.SetAztecOption(AZ_output, 32);
solver2.SetAztecOption(AZ_kspace, 160);
solver2.Iterate(1550, 1e-12);
// ==================================================== //
// compute difference between the two ML solutions //
// ==================================================== //
double d = 0.0, d_tot = 0.0;
for( int i=0 ; i<lhs->Map().NumMyElements() ; ++i )
d += ((*lhs)[0][i] - lhs2[0][i]) * ((*lhs)[0][i] - lhs2[0][i]);
A->Comm().SumAll(&d,&d_tot,1);
string msg = ProblemType;
if (A->Comm().MyPID() == 0) {
cout << msg << "......Using " << A->Comm().NumProc() << " processes" << endl;
cout << msg << "......||x_1 - x_2||_2 = " << sqrt(d_tot) << endl;
cout << msg << "......Total Time = " << Time.ElapsedTime() << endl;
}
TotalErrorExactSol += sqrt(d_tot);
return( solver.NumIters() );
}
int main(int argc, char** argv) {
#if defined(HAVE_MPI) && defined(HAVE_EPETRA)
int p, numProcs = 1;
int localProc = 0;
//first, set up our MPI environment...
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &localProc);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
int local_n = 600;
//Create a Epetra_LinearProblem object.
Epetra_LinearProblem* linprob = 0;
try {
linprob = create_epetra_problem(numProcs, localProc, local_n);
}
catch(std::exception& exc) {
std::cout << "linsys example: create_epetra_problem threw exception '"
<< exc.what() << "' on proc " << localProc << std::endl;
MPI_Finalize();
return(-1);
}
//We'll need a Teuchos::ParameterList object to pass to the
//Isorropia::Epetra::Partitioner class.
Teuchos::ParameterList paramlist;
// If Zoltan is available, the Zoltan package will be used for
// the partitioning operation. By default, Isorropia selects Zoltan's
// Hypergraph partitioner. If a method other than Hypergraph is
// desired, it can be specified by first creating a parameter sublist
// named "Zoltan", and then setting appropriate Zoltan parameters in
// that sublist. A sublist is created like this:
// Teuchos::ParameterList& sublist = paramlist.sublist("Zoltan");
//
// If Zoltan is not available, a simple linear partitioner will be
// used to partition such that the number of nonzeros is equal (or
// close to equal) on each processor.
Epetra_RowMatrix* rowmatrix = linprob->GetMatrix();
Teuchos::RCP<const Epetra_RowMatrix> rowmat =
Teuchos::rcp(rowmatrix, false);
//Now create the partitioner
Teuchos::RCP<Isorropia::Epetra::Partitioner> partitioner =
Teuchos::rcp(new Isorropia::Epetra::Partitioner(rowmat, paramlist));
//Next create a Redistributor object and use it to create balanced
//copies of the objects in linprob.
Isorropia::Epetra::Redistributor rd(partitioner);
Teuchos::RCP<Epetra_CrsMatrix> bal_matrix;
Teuchos::RCP<Epetra_MultiVector> bal_x;
Teuchos::RCP<Epetra_MultiVector> bal_b;
//Use a try-catch block because Isorropia will throw an exception
//if it encounters an error.
if (localProc == 0) {
std::cout << " calling Isorropia::Epetra::Redistributor::redistribute..."
<< std::endl;
}
try {
bal_matrix = rd.redistribute(*linprob->GetMatrix());
bal_x = rd.redistribute(*linprob->GetLHS());
bal_b = rd.redistribute(*linprob->GetRHS());
}
catch(std::exception& exc) {
std::cout << "linsys example: Isorropia::Epetra::Redistributor threw "
<< "exception '" << exc.what() << "' on proc "
<< localProc << std::endl;
MPI_Finalize();
return(-1);
}
Epetra_LinearProblem balanced_problem(bal_matrix.get(),
bal_x.get(), bal_b.get());
// Results
double goalWeight = 1.0 / (double)numProcs;
double bal0, bal1, cutn0, cutn1, cutl0, cutl1;
Isorropia::Epetra::CostDescriber default_costs;
// Balance and cut quality before partitioning
ispatest::compute_hypergraph_metrics(*(linprob->GetMatrix()), default_costs, goalWeight,
bal0, cutn0, cutl0);
// Balance and cut quality after partitioning
ispatest::compute_hypergraph_metrics(*bal_matrix, default_costs, goalWeight,
bal1, cutn1, cutl1);
if (localProc == 0){
std::cout << "Before partitioning: ";
std::cout << "Balance " << bal0 << " cutN " << cutn0 << " cutL " << cutl0;
std::cout << std::endl;
std::cout << "After partitioning: ";
std::cout << "Balance " << bal1 << " cutN " << cutn1 << " cutL " << cutl1;
std::cout << std::endl;
}
//Finally, delete the pointer objects that we asked to be created.
delete linprob->GetMatrix();
delete linprob->GetLHS();
delete linprob->GetRHS();
delete linprob;
if (localProc == 0) {
std::cout << std::endl;
}
MPI_Finalize();
#else
std::cout << "part_redist: must have both MPI and EPETRA. Make sure Trilinos "
<< "is configured with --enable-mpi and --enable-epetra." << std::endl;
#endif
return(0);
}
