void AmesosLinearOpWithSolve::describe(
  Teuchos::FancyOStream &out,
  const Teuchos::EVerbosityLevel verbLevel
  ) const
{
  using Teuchos::OSTab;
  using Teuchos::typeName;
  using Teuchos::describe;
  switch(verbLevel) {
    case Teuchos::VERB_DEFAULT:
    case Teuchos::VERB_LOW:
      out << this->description() << std::endl;
      break;
    case Teuchos::VERB_MEDIUM:
    case Teuchos::VERB_HIGH:
    case Teuchos::VERB_EXTREME:
    {
      out
        << Teuchos::Describable::description() << "{"
        << "rangeDim=" << this->range()->dim()
        << ",domainDim="<< this->domain()->dim() << "}\n";
      OSTab tab(out);
      if(!is_null(fwdOp_)) {
        out << "fwdOp = " << describe(*fwdOp_,verbLevel);
      }
      if(!is_null(amesosSolver_)) {
        out << "amesosSolver=" << typeName(*amesosSolver_) << "\n";
      }
      break;
    }
    default:
      TEST_FOR_EXCEPT(true); // Should never get here!
  }
}
void BelosLinearOpWithSolve<Scalar>::describe(
  Teuchos::FancyOStream &out_arg,
  const Teuchos::EVerbosityLevel verbLevel
  ) const
{
  typedef Teuchos::ScalarTraits<Scalar> ST;
  using Teuchos::FancyOStream;
  using Teuchos::OSTab;
  using Teuchos::describe;
  RCP<FancyOStream> out = rcp(&out_arg,false);
  OSTab tab(out);
  switch (verbLevel) {
    case Teuchos::VERB_DEFAULT:
    case Teuchos::VERB_LOW:
      *out << this->description() << std::endl;
      break;
    case Teuchos::VERB_MEDIUM:
    case Teuchos::VERB_HIGH:
    case Teuchos::VERB_EXTREME:
    {
      *out
        << Teuchos::Describable::description()<< "{"
        << "rangeDim=" << this->range()->dim()
        << ",domainDim=" << this->domain()->dim() << "}\n";
      if (lp_->getOperator().get()) {
        OSTab tab(out);
        *out
          << "iterativeSolver = "<<describe(*iterativeSolver_,verbLevel)
          << "fwdOp = " << describe(*lp_->getOperator(),verbLevel);
        if (lp_->getLeftPrec().get())
          *out << "leftPrecOp = "<<describe(*lp_->getLeftPrec(),verbLevel);
        if (lp_->getRightPrec().get())
          *out << "rightPrecOp = "<<describe(*lp_->getRightPrec(),verbLevel);
      }
      break;
    }
    default:
      TEST_FOR_EXCEPT(true); // Should never get here!
  }
}
void DefaultProductVector<Scalar>::describe(
  Teuchos::FancyOStream &out_arg,
  const Teuchos::EVerbosityLevel verbLevel
  ) const
{
  typedef Teuchos::ScalarTraits<Scalar>  ST;
  using Teuchos::FancyOStream;
  using Teuchos::OSTab;
  using Teuchos::describe;
  RCP<FancyOStream> out = rcp(&out_arg,false);
  OSTab tab(out);
  switch(verbLevel) {
    case Teuchos::VERB_NONE:
      break;
    case Teuchos::VERB_DEFAULT:
    case Teuchos::VERB_LOW:
      *out << this->description() << std::endl;
      break;
    case Teuchos::VERB_MEDIUM:
    case Teuchos::VERB_HIGH:
    case Teuchos::VERB_EXTREME:
    {
      *out
        << Teuchos::Describable::description() << "{"
        << "dim=" << this->space()->dim()
        << "}\n";
      OSTab tab2(out);
      *out
        <<  "numBlocks="<< numBlocks_ << std::endl
        <<  "Constituent vector objects v[0], v[1], ... v[numBlocks-1]:\n";
      OSTab tab3(out);
      for( int k = 0; k < numBlocks_; ++k ) {
        *out << "v["<<k<<"] = " << describe(*vecs_[k].getConstObj(),verbLevel);
      }
      break;
    }
    default:
      TEST_FOR_EXCEPT(true); // Should never get here!
  }
}
SolveStatus<Scalar>
BelosLinearOpWithSolve<Scalar>::solveImpl(
  const EOpTransp M_trans,
  const MultiVectorBase<Scalar> &B,
  const Ptr<MultiVectorBase<Scalar> > &X,
  const Ptr<const SolveCriteria<Scalar> > solveCriteria
  ) const
{

  TEUCHOS_FUNC_TIME_MONITOR("BelosLOWS");

  using Teuchos::rcp;
  using Teuchos::rcpFromRef;
  using Teuchos::rcpFromPtr;
  using Teuchos::FancyOStream;
  using Teuchos::OSTab;
  using Teuchos::describe;
  typedef Teuchos::ScalarTraits<Scalar> ST;
  typedef typename ST::magnitudeType ScalarMag;
  Teuchos::Time totalTimer(""), timer("");
  totalTimer.start(true);

  assertSolveSupports(*this, M_trans, solveCriteria);
  // 2010/08/22: rabartl: Bug 4915 ToDo: Move the above into the NIV function
  // solve(...).

  const int numRhs = B.domain()->dim();
  const int numEquations = B.range()->dim();

  const RCP<FancyOStream> out = this->getOStream();
  const Teuchos::EVerbosityLevel verbLevel = this->getVerbLevel();
  OSTab tab = this->getOSTab();
  if (out.get() && static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_LOW)) {
    *out << "\nStarting iterations with Belos:\n";
    OSTab tab2(out);
    *out << "Using forward operator = " << describe(*fwdOpSrc_->getOp(),verbLevel);
    *out << "Using iterative solver = " << describe(*iterativeSolver_,verbLevel);
    *out << "With #Eqns="<<numEquations<<", #RHSs="<<numRhs<<" ...\n";
  }

  //
  // Set RHS and LHS
  //

  bool ret = lp_->setProblem( rcpFromPtr(X), rcpFromRef(B) );
  TEST_FOR_EXCEPTION(
    ret == false, CatastrophicSolveFailure
    ,"Error, the Belos::LinearProblem could not be set for the current solve!"
    );

  //
  // Set the solution criteria
  //

  const RCP<Teuchos::ParameterList> tmpPL = Teuchos::parameterList();

  SolveMeasureType solveMeasureType;
  RCP<GeneralSolveCriteriaBelosStatusTest<Scalar> > generalSolveCriteriaBelosStatusTest;
  if (nonnull(solveCriteria)) {
    solveMeasureType = solveCriteria->solveMeasureType;
    const ScalarMag requestedTol = solveCriteria->requestedTol;
    if (solveMeasureType.useDefault()) {
      tmpPL->set("Convergence Tolerance", defaultTol_);
    }
    else if (solveMeasureType(SOLVE_MEASURE_NORM_RESIDUAL, SOLVE_MEASURE_NORM_RHS)) {
      if (requestedTol != SolveCriteria<Scalar>::unspecifiedTolerance()) {
        tmpPL->set("Convergence Tolerance", requestedTol);
      }
      else {
        tmpPL->set("Convergence Tolerance", defaultTol_);
      }
      tmpPL->set("Explicit Residual Scaling", "Norm of RHS");
    }
    else if (solveMeasureType(SOLVE_MEASURE_NORM_RESIDUAL, SOLVE_MEASURE_NORM_INIT_RESIDUAL)) {
      if (requestedTol != SolveCriteria<Scalar>::unspecifiedTolerance()) {
        tmpPL->set("Convergence Tolerance", requestedTol);
      }
      else {
        tmpPL->set("Convergence Tolerance", defaultTol_);
      }
      tmpPL->set("Explicit Residual Scaling", "Norm of Initial Residual");
    }
    else {
      // Set the most generic (and inefficient) solve criteria
      generalSolveCriteriaBelosStatusTest = createGeneralSolveCriteriaBelosStatusTest(
        *solveCriteria, convergenceTestFrequency_);
      // Set the verbosity level (one level down)
      generalSolveCriteriaBelosStatusTest->setOStream(out);
      generalSolveCriteriaBelosStatusTest->setVerbLevel(incrVerbLevel(verbLevel, -1));
      // Set the default convergence tolerance to always converged to allow
      // the above status test to control things.
      tmpPL->set("Convergence Tolerance", 1.0);
    }
  }
  else {
    // No solveCriteria was even passed in!
    tmpPL->set("Convergence Tolerance", defaultTol_);
  }

  //
  // Reset the blocksize if we adding more vectors than half the number of equations,
  // orthogonalization will fail on the first iteration!
  //

  RCP<const Teuchos::ParameterList> solverParams = iterativeSolver_->getCurrentParameters();
  const int currBlockSize = Teuchos::getParameter<int>(*solverParams, "Block Size");
  bool isNumBlocks = false;
  int currNumBlocks = 0;
  if (Teuchos::isParameterType<int>(*solverParams, "Num Blocks")) {
    currNumBlocks = Teuchos::getParameter<int>(*solverParams, "Num Blocks");
    isNumBlocks = true;
  }
  const int newBlockSize = TEUCHOS_MIN(currBlockSize,numEquations/2);
  if (nonnull(out)
    && static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_NONE)
    && newBlockSize != currBlockSize)
  {
    *out << "\nAdjusted block size = " << newBlockSize << "\n";
  }
  //
  tmpPL->set("Block Size",newBlockSize);

  //
  // Set the number of Krylov blocks if we are using a GMRES solver, or a solver
  // that recognizes "Num Blocks". Otherwise the solver will throw an error!
  //

  if (isNumBlocks) {
    const int Krylov_length = (currNumBlocks*currBlockSize)/newBlockSize;
    tmpPL->set("Num Blocks",Krylov_length);
  
    if (newBlockSize != currBlockSize) {
      if (out.get() && static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_NONE))
        *out
          << "\nAdjusted max number of Krylov basis blocks = " << Krylov_length << "\n";
    }
  }

  //
  // Solve the linear system
  //

  Belos::ReturnType belosSolveStatus;
  {
    RCP<std::ostream>
      outUsed =
      ( static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_NONE)
        ? out
        : rcp(new FancyOStream(rcp(new Teuchos::oblackholestream())))
        );
    Teuchos::OSTab tab(outUsed,1,"BELOS");
    tmpPL->set("Output Stream", outUsed);
    iterativeSolver_->setParameters(tmpPL);
    if (nonnull(generalSolveCriteriaBelosStatusTest)) {
      iterativeSolver_->setUserConvStatusTest(generalSolveCriteriaBelosStatusTest);
    }
    belosSolveStatus = iterativeSolver_->solve();
  }

  //
  // Report the solve status
  //

  totalTimer.stop();

  SolveStatus<Scalar> solveStatus;

  switch (belosSolveStatus) {
    case Belos::Unconverged: {
      solveStatus.solveStatus = SOLVE_STATUS_UNCONVERGED;
      break;
    }
    case Belos::Converged: {
      solveStatus.solveStatus = SOLVE_STATUS_CONVERGED;
      if (nonnull(generalSolveCriteriaBelosStatusTest)) {
        const ArrayView<const ScalarMag> achievedTol = 
          generalSolveCriteriaBelosStatusTest->achievedTol();
        solveStatus.achievedTol = ST::zero();
        for (Ordinal i = 0; i < achievedTol.size(); ++i) {
          solveStatus.achievedTol = std::max(solveStatus.achievedTol, achievedTol[i]);
        }
      }
      else {
        solveStatus.achievedTol = tmpPL->get("Convergence Tolerance", defaultTol_);
      }
      break;
    }
    TEUCHOS_SWITCH_DEFAULT_DEBUG_ASSERT();
  }

  std::ostringstream ossmessage;
  ossmessage
    << "The Belos solver of type \""<<iterativeSolver_->description()
    <<"\" returned a solve status of \""<< toString(solveStatus.solveStatus) << "\""
    << " in " << iterativeSolver_->getNumIters() << " iterations"
    << " with total CPU time of " << totalTimer.totalElapsedTime() << " sec" ;
  if (out.get() && static_cast<int>(verbLevel) >=static_cast<int>(Teuchos::VERB_LOW))
    *out << "\n" << ossmessage.str() << "\n";

  solveStatus.message = ossmessage.str();

  if (out.get() && static_cast<int>(verbLevel) >= static_cast<int>(Teuchos::VERB_LOW))
    *out << "\nTotal solve time in Belos = "<<totalTimer.totalElapsedTime()<<" sec\n";

  return solveStatus;

}
bool VectorTester<Scalar>::check(
  const VectorBase<Scalar>       &v
  ,Teuchos::FancyOStream         *out_arg
  ) const
{

  using std::endl;
  using Teuchos::describe;
  using Teuchos::FancyOStream;
  using Teuchos::OSTab;
  typedef Teuchos::ScalarTraits<Scalar> ST;
  //typedef typename ST::magnitudeType    ScalarMag;

  Teuchos::RCP<FancyOStream> out = Teuchos::rcp(out_arg,false);
  const Teuchos::EVerbosityLevel verbLevel = (dump_all()?Teuchos::VERB_EXTREME:Teuchos::VERB_MEDIUM);

  OSTab tab(out,1,"THYRA");

  bool result, success = true;

  if(out.get()) *out <<endl<< "*** Entering Thyra::VectorTester<"<<ST::name()<<">::check(v,...) ...\n";

  if(out.get()) *out <<endl<< "Testing a VectorBase object described as:\n" << describe(v,verbLevel);

  if(out.get()) *out <<endl<< "A) Creating temporary vector t1, t2, t3, and t4 from v.space() ...\n";
  Teuchos::RCP<const Thyra::VectorSpaceBase<Scalar> >
    vs = v.space();
  Teuchos::RCP<Thyra::VectorBase<Scalar> >
    t1 = createMember(vs), t2 = createMember(vs), t3 = createMember(vs), t4 = createMember(vs);

  if(out.get()) *out <<endl<< "B) Testing VectorBase::applyOp(...) by calling a few standard RTOp operations ... ";

  const Scalar
    one   = ST::one(),
    two   = Scalar(2)*one,
    three = Scalar(3)*one;

  {
    using Teuchos::inoutArg;

    TestResultsPrinter testResultsPrinter(out, show_all_tests());
    const RCP<FancyOStream> testOut = testResultsPrinter.getTestOStream();

    bool these_results = true;
    
    *testOut <<endl<< "assign(t1.ptr(),2.0) ...\n";
    Thyra::assign( t1.ptr(), two );
    if(dump_all()) *testOut <<endl<< "\nt1 =\n" << describe(*t1,verbLevel);
    
    result = Teuchos::testRelErr<Scalar>(
      "sum(t1)", sum(*t1), "2*vs->dim()", two*Scalar(vs->dim()),
      "error_tol()", error_tol(), "warning_tol()", warning_tol(),
      inoutArg(*testOut)
      );
    if(!result) these_results = false;
    
    *testOut <<endl<< "assign(t2.ptr(),3.0) ...\n";
    Thyra::assign( t2.ptr(), three );
    if(dump_all()) *testOut <<endl<< "t2 =\n" << *t1;
    
    result = Teuchos::testRelErr<Scalar>(
      "sum(t2)",sum(*t2),"3*vs->dim()",three*Scalar(vs->dim()),
      "error_tol()",error_tol(),"warning_tol()",warning_tol(),
      inoutArg(*testOut)
      );
    if(!result) these_results = false;
    
    result = Teuchos::testRelErr<Scalar>(
      "vs->scalarProd(*t1,*t2)",vs->scalarProd(*t1,*t2),"2*3*vs->dim()",two*three*Scalar(vs->dim()),
      "error_tol()",error_tol(),"warning_tol()",warning_tol(),
      inoutArg(*testOut)
      );
    if(!result) these_results = false;

    testResultsPrinter.printTestResults(these_results, inoutArg(success));

  }
    
  // ToDo: Test the rest of the specific VectorBase interface on v1

  if(out.get()) *out <<endl<< "C) Checking the MultiVectorBase interface of v ...\n";
  result = multiVectorTester_.check(v, out.ptr());
  if(!result) success = false;

  if(out.get()) *out <<endl<< "*** Leaving Thyra::VectorTester<"<<ST::name()<<">::check(v,...) ...\n";
  
  return success;

}
SolveStatus<Scalar>
BelosLinearOpWithSolve<Scalar>::solveImpl(
  const EOpTransp M_trans,
  const MultiVectorBase<Scalar> &B,
  const Ptr<MultiVectorBase<Scalar> > &X,
  const Ptr<const SolveCriteria<Scalar> > solveCriteria
  ) const
{

  THYRA_FUNC_TIME_MONITOR("Stratimikos: BelosLOWS");

  using Teuchos::rcp;
  using Teuchos::rcpFromRef;
  using Teuchos::rcpFromPtr;
  using Teuchos::FancyOStream;
  using Teuchos::OSTab;
  using Teuchos::ParameterList;
  using Teuchos::parameterList;
  using Teuchos::describe;
  typedef Teuchos::ScalarTraits<Scalar> ST;
  typedef typename ST::magnitudeType ScalarMag;
  Teuchos::Time totalTimer(""), timer("");
  totalTimer.start(true);

  assertSolveSupports(*this, M_trans, solveCriteria);
  // 2010/08/22: rabartl: Bug 4915 ToDo: Move the above into the NIV function
  // solve(...).

  const RCP<FancyOStream> out = this->getOStream();
  const Teuchos::EVerbosityLevel verbLevel = this->getVerbLevel();
  OSTab tab = this->getOSTab();
  if (out.get() && static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_LOW)) {
    *out << "\nStarting iterations with Belos:\n";
    OSTab tab2(out);
    *out << "Using forward operator = " << describe(*fwdOpSrc_->getOp(),verbLevel);
    *out << "Using iterative solver = " << describe(*iterativeSolver_,verbLevel);
    *out << "With #Eqns="<<B.range()->dim()<<", #RHSs="<<B.domain()->dim()<<" ...\n";
  }

  //
  // Set RHS and LHS
  //

  bool ret = lp_->setProblem( rcpFromPtr(X), rcpFromRef(B) );
  TEUCHOS_TEST_FOR_EXCEPTION(
    ret == false, CatastrophicSolveFailure
    ,"Error, the Belos::LinearProblem could not be set for the current solve!"
    );

  //
  // Set the solution criteria
  //

  // Parameter list for the current solve.
  const RCP<ParameterList> tmpPL = Teuchos::parameterList();

  // The solver's valid parameter list.
  RCP<const ParameterList> validPL = iterativeSolver_->getValidParameters();

  SolveMeasureType solveMeasureType;
  RCP<GeneralSolveCriteriaBelosStatusTest<Scalar> > generalSolveCriteriaBelosStatusTest;
  if (nonnull(solveCriteria)) {
    solveMeasureType = solveCriteria->solveMeasureType;
    const ScalarMag requestedTol = solveCriteria->requestedTol;
    if (solveMeasureType.useDefault()) {
      tmpPL->set("Convergence Tolerance", defaultTol_);
    }
    else if (solveMeasureType(SOLVE_MEASURE_NORM_RESIDUAL, SOLVE_MEASURE_NORM_RHS)) {
      if (requestedTol != SolveCriteria<Scalar>::unspecifiedTolerance()) {
        tmpPL->set("Convergence Tolerance", requestedTol);
      }
      else {
        tmpPL->set("Convergence Tolerance", defaultTol_);
      }
      setResidualScalingType (tmpPL, validPL, "Norm of RHS");
    }
    else if (solveMeasureType(SOLVE_MEASURE_NORM_RESIDUAL, SOLVE_MEASURE_NORM_INIT_RESIDUAL)) {
      if (requestedTol != SolveCriteria<Scalar>::unspecifiedTolerance()) {
        tmpPL->set("Convergence Tolerance", requestedTol);
      }
      else {
        tmpPL->set("Convergence Tolerance", defaultTol_);
      }
      setResidualScalingType (tmpPL, validPL, "Norm of Initial Residual");
    }
    else {
      // Set the most generic (and inefficient) solve criteria
      generalSolveCriteriaBelosStatusTest = createGeneralSolveCriteriaBelosStatusTest(
        *solveCriteria, convergenceTestFrequency_);
      // Set the verbosity level (one level down)
      generalSolveCriteriaBelosStatusTest->setOStream(out);
      generalSolveCriteriaBelosStatusTest->setVerbLevel(incrVerbLevel(verbLevel, -1));
      // Set the default convergence tolerance to always converged to allow
      // the above status test to control things.
      tmpPL->set("Convergence Tolerance", 1.0);
    }
    // maximum iterations
    if (nonnull(solveCriteria->extraParameters)) {
      if (Teuchos::isParameterType<int>(*solveCriteria->extraParameters,"Maximum Iterations")) {
        tmpPL->set("Maximum Iterations", Teuchos::get<int>(*solveCriteria->extraParameters,"Maximum Iterations"));
      }
    }
  }
  else {
    // No solveCriteria was even passed in!
    tmpPL->set("Convergence Tolerance", defaultTol_);
  }

  //
  // Solve the linear system
  //

  Belos::ReturnType belosSolveStatus;
  {
    RCP<std::ostream>
      outUsed =
      ( static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_LOW)
        ? out
        : rcp(new FancyOStream(rcp(new Teuchos::oblackholestream())))
        );
    Teuchos::OSTab tab1(outUsed,1,"BELOS");
    tmpPL->set("Output Stream", outUsed);
    iterativeSolver_->setParameters(tmpPL);
    if (nonnull(generalSolveCriteriaBelosStatusTest)) {
      iterativeSolver_->setUserConvStatusTest(generalSolveCriteriaBelosStatusTest);
    }
    belosSolveStatus = iterativeSolver_->solve();
  }

  //
  // Report the solve status
  //

  totalTimer.stop();

  SolveStatus<Scalar> solveStatus;

  switch (belosSolveStatus) {
    case Belos::Unconverged: {
      solveStatus.solveStatus = SOLVE_STATUS_UNCONVERGED;
      // Set achievedTol even if the solver did not converge.  This is
      // helpful for things like nonlinear solvers, which might be
      // able to use a partially converged result, and which would
      // like to know the achieved convergence tolerance for use in
      // computing bounds.  It's also helpful for estimating whether a
      // small increase in the maximum iteration count might be
      // helpful next time.
      try {
	// Some solvers might not have implemented achievedTol(). 
	// The default implementation throws std::runtime_error.
	solveStatus.achievedTol = iterativeSolver_->achievedTol();
      } catch (std::runtime_error&) {
	// Do nothing; use the default value of achievedTol.
      }
      break;
    }
    case Belos::Converged: {
      solveStatus.solveStatus = SOLVE_STATUS_CONVERGED;
      if (nonnull(generalSolveCriteriaBelosStatusTest)) {
	// The user set a custom status test.  This means that we
	// should ask the custom status test itself, rather than the
	// Belos solver, what the final achieved convergence tolerance
	// was.
        const ArrayView<const ScalarMag> achievedTol = 
          generalSolveCriteriaBelosStatusTest->achievedTol();
        solveStatus.achievedTol = ST::zero();
        for (Ordinal i = 0; i < achievedTol.size(); ++i) {
          solveStatus.achievedTol = std::max(solveStatus.achievedTol, achievedTol[i]);
        }
      }
      else {
	try {
	  // Some solvers might not have implemented achievedTol(). 
	  // The default implementation throws std::runtime_error.
	  solveStatus.achievedTol = iterativeSolver_->achievedTol();
	} catch (std::runtime_error&) {
	  // Use the default convergence tolerance.  This is a correct
	  // upper bound, since we did actually converge.
	  solveStatus.achievedTol = tmpPL->get("Convergence Tolerance", defaultTol_);
	}
      }
      break;
    }
    TEUCHOS_SWITCH_DEFAULT_DEBUG_ASSERT();
  }

  std::ostringstream ossmessage;
  ossmessage
    << "The Belos solver of type \""<<iterativeSolver_->description()
    <<"\" returned a solve status of \""<< toString(solveStatus.solveStatus) << "\""
    << " in " << iterativeSolver_->getNumIters() << " iterations"
    << " with total CPU time of " << totalTimer.totalElapsedTime() << " sec" ;
  if (out.get() && static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_NONE))
    *out << "\n" << ossmessage.str() << "\n";

  solveStatus.message = ossmessage.str();

  // Dump the getNumIters() and the achieved convergence tolerance
  // into solveStatus.extraParameters, as the "Belos/Iteration Count"
  // resp. "Belos/Achieved Tolerance" parameters.
  if (solveStatus.extraParameters.is_null()) {
    solveStatus.extraParameters = parameterList ();
  }
  solveStatus.extraParameters->set ("Belos/Iteration Count", 
				    iterativeSolver_->getNumIters());\
  // package independent version of the same
  solveStatus.extraParameters->set ("Iteration Count", 
				    iterativeSolver_->getNumIters());\
  // NOTE (mfh 13 Dec 2011) Though the most commonly used Belos
  // solvers do implement achievedTol(), some Belos solvers currently
  // do not.  In the latter case, if the solver did not converge, the
  // reported achievedTol() value may just be the default "invalid"
  // value -1, and if the solver did converge, the reported value will
  // just be the convergence tolerance (a correct upper bound).
  solveStatus.extraParameters->set ("Belos/Achieved Tolerance", 
				    solveStatus.achievedTol);

//  This information is in the previous line, which is printed anytime the verbosity
//  is not set to Teuchos::VERB_NONE, so I'm commenting this out for now.
//  if (out.get() && static_cast<int>(verbLevel) > static_cast<int>(Teuchos::VERB_NONE))
//    *out << "\nTotal solve time in Belos = "<<totalTimer.totalElapsedTime()<<" sec\n";
  
  return solveStatus;

}
TEUCHOS_UNIT_TEST( BasicDiscreteAdjointStepperTester, rawNonlinearAdjoint )
{

  using Teuchos::outArg;
  using Teuchos::describe;
  using Teuchos::getParametersFromXmlString;
  typedef Thyra::ModelEvaluatorBase MEB;

  //
  out << "\nA) Create the nonlinear ME ...\n";
  //

  RCP<VanderPolModel> stateModel = vanderPolModel(
    getParametersFromXmlString(
      "<ParameterList>"
      "  <Parameter name=\"Implicit model formulation\" type=\"bool\" value=\"1\"/>"
      "</ParameterList>"
      )
    );

  //
  out << "\nB) Create the nonlinear solver ...\n";
  //

  RCP<TimeStepNonlinearSolver<double> > nlSolver = timeStepNonlinearSolver<double>(
    getParametersFromXmlString(
      "<ParameterList>"
      "  <Parameter name=\"Default Tol\" type=\"double\" value=\"1.0e-10\"/>"
      "  <Parameter name=\"Default Max Iters\" type=\"int\" value=\"20\"/>"
      "</ParameterList>"
      )
    );

  //
  out << "\nC) Create the integrator for the forward state problem ...\n";
  //

  RCP<IntegratorBuilder<double> > ib = integratorBuilder<double>(
    Teuchos::getParametersFromXmlString(
      "<ParameterList>"
      "  <ParameterList name=\"Stepper Settings\">"
      "    <ParameterList name=\"Stepper Selection\">"
      "      <Parameter name=\"Stepper Type\" type=\"string\" value=\"Backward Euler\"/>"
      "    </ParameterList>"
      "  </ParameterList>"
      "  <ParameterList name=\"Integration Control Strategy Selection\">"
      "    <Parameter name=\"Integration Control Strategy Type\" type=\"string\""
      "      value=\"Simple Integration Control Strategy\"/>"
      "    <ParameterList name=\"Simple Integration Control Strategy\">"
      "      <Parameter name=\"Take Variable Steps\" type=\"bool\" value=\"false\"/>"
      "      <Parameter name=\"Fixed dt\" type=\"double\" value=\"0.5\"/>" // Gives 2 time steps!
      "    </ParameterList>"
      "  </ParameterList>"
      "  <ParameterList name=\"Interpolation Buffer Settings\">"
      "    <ParameterList name=\"Trailing Interpolation Buffer Selection\">"
      "      <Parameter name=\"Interpolation Buffer Type\" type=\"string\" value=\"Interpolation Buffer\"/>"
      "    </ParameterList>"
      "  </ParameterList>"
      "</ParameterList>"
      )
    );
  
  MEB::InArgs<double> ic = stateModel->getNominalValues();
  RCP<IntegratorBase<double> > integrator = ib->create(stateModel, ic, nlSolver);
  //integrator->setVerbLevel(Teuchos::VERB_EXTREME);

  // ToDo: Set the trailing IB to pick up the entire state solution!

  // 
  out << "\nD) Solve the basic forward problem ...\n";
  //

  const TimeRange<double> fwdTimeRange = integrator->getFwdTimeRange();
  const double t_final = fwdTimeRange.upper();
  RCP<const Thyra::VectorBase<double> > x_final, x_dot_final;
  get_fwd_x_and_x_dot( *integrator, t_final, outArg(x_final), outArg(x_dot_final) );

  out << "\nt_final = " << t_final << "\n";
  out << "\nx_final: " << *x_final;
  out << "\nx_dot_final: " << *x_dot_final;

  //
  out << "\nE) Create the basic adjoint model (no distributed response) ...\n";
  //

  RCP<AdjointModelEvaluator<double> > adjModel =
    adjointModelEvaluator<double>(
      stateModel, fwdTimeRange
      );
  adjModel->setFwdStateSolutionBuffer(integrator);

  //
  out << "\nF) Create a stepper and integrator for the adjoint ...\n";
  //
  
  RCP<Thyra::LinearNonlinearSolver<double> > adjTimeStepSolver =
    Thyra::linearNonlinearSolver<double>();
  RCP<Rythmos::StepperBase<double> > adjStepper =
    integrator->getStepper()->cloneStepperAlgorithm();

  //
  out << "\nG) Set up the initial condition for the adjoint at the final time ...\n";
  //
  
  const RCP<const Thyra::VectorSpaceBase<double> >
    f_space = stateModel->get_f_space();
  
  // lambda(t_final) = x_final
  const RCP<Thyra::VectorBase<double> > lambda_ic = createMember(f_space);
  V_V( lambda_ic.ptr(), *x_final );
  
  // lambda_dot(t_final,i) = 0.0
  const RCP<Thyra::VectorBase<double> > lambda_dot_ic = createMember(f_space);
  Thyra::V_S( lambda_dot_ic.ptr(), 0.0 );
  
  MEB::InArgs<double> adj_ic = adjModel->getNominalValues();
  adj_ic.set_x(lambda_ic);
  adj_ic.set_x_dot(lambda_dot_ic);
  out << "\nadj_ic: " << describe(adj_ic, Teuchos::VERB_EXTREME);

  RCP<Rythmos::IntegratorBase<double> > adjIntegrator =
    ib->create(adjModel, adj_ic, adjTimeStepSolver);
  
  //
  out << "\nH) Integrate the adjoint backwards in time (using backward time) ...\n";
  //
  
  adjStepper->setInitialCondition(adj_ic);
  adjIntegrator->setStepper(adjStepper, fwdTimeRange.length());
  
  const double adj_t_final = fwdTimeRange.length();
  RCP<const Thyra::VectorBase<double> > lambda_final, lambda_dot_final;
  get_fwd_x_and_x_dot( *adjIntegrator, adj_t_final,
    outArg(lambda_final), outArg(lambda_dot_final) );

  out << "\nadj_t_final = " << adj_t_final << "\n";
  out << "\nlambda_final: " << *lambda_final;
  out << "\nlambda_dot_final: " << *lambda_dot_final;

}
int main(int argc, char *argv[])
{

  using std::endl;
  typedef double Scalar;
  // typedef double ScalarMag; // unused
  typedef Teuchos::ScalarTraits<Scalar> ST;
  using Teuchos::describe;
  using Teuchos::Array;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::outArg;
  using Teuchos::rcp_implicit_cast;
  using Teuchos::rcp_dynamic_cast;
  using Teuchos::as;
  using Teuchos::ParameterList;
  using Teuchos::CommandLineProcessor;
  typedef Teuchos::ParameterList::PrintOptions PLPrintOptions;
  typedef Thyra::ModelEvaluatorBase MEB;

  bool result, success = true;

  Teuchos::GlobalMPISession mpiSession(&argc,&argv);

  RCP<Epetra_Comm> epetra_comm;
#ifdef HAVE_MPI
  epetra_comm = rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
  epetra_comm = rcp( new Epetra_SerialComm );
#endif // HAVE_MPI

  RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

  try {

    //
    // Read commandline options
    //

    CommandLineProcessor clp;
    clp.throwExceptions(false);
    clp.addOutputSetupOptions(true);

    std::string paramsFileName = "";
    clp.setOption( "params-file", &paramsFileName,
      "File name for XML parameters" );

    double t_final = 1e-3;
    clp.setOption( "final-time", &t_final,
      "Final integration time (initial time is 0.0)" );

    int numTimeSteps = 10;
    clp.setOption( "num-time-steps", &numTimeSteps,
      "Number of (fixed) time steps.  If <= 0.0, then variable time steps are taken" );

    double maxStateError = 1e-14;
    clp.setOption( "max-state-error", &maxStateError,
      "Maximum relative error in the integrated state allowed" );

    Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_DEFAULT;
    setVerbosityLevelOption( "verb-level", &verbLevel,
      "Top-level verbosity level.  By default, this gets deincremented as you go deeper into numerical objects.",
      &clp );

    Teuchos::EVerbosityLevel solnVerbLevel = Teuchos::VERB_DEFAULT;
    setVerbosityLevelOption( "soln-verb-level", &solnVerbLevel,
      "Solution verbosity level",
      &clp );

    CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
    if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;

    //
    *out << "\nA) Get the base parameter list ...\n";
    //

    RCP<ParameterList>
      paramList = Teuchos::parameterList();
    if (paramsFileName.length())
      updateParametersFromXmlFile( paramsFileName, paramList.ptr() );

    paramList->validateParameters(*getValidParameters());

    const Scalar t_init = 0.0;

    const Rythmos::TimeRange<Scalar> fwdTimeRange(t_init, t_final);
    const Scalar delta_t = t_final / numTimeSteps;
    *out << "\ndelta_t = " << delta_t;

    //
    *out << "\nB) Create the Stratimikos linear solver factory ...\n";
    //
    // This is the linear solve strategy that will be used to solve for the
    // linear system with the W.
    //

    Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
    linearSolverBuilder.setParameterList(sublist(paramList,Stratimikos_name));
    RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
      W_factory = createLinearSolveStrategy(linearSolverBuilder);

    //
    *out << "\nC) Create and initalize the forward model ...\n";
    //

    // C.1) Create the underlying EpetraExt::ModelEvaluator

    RCP<EpetraExt::DiagonalTransientModel> epetraStateModel =
      EpetraExt::diagonalTransientModel(
        epetra_comm,
        sublist(paramList,DiagonalTransientModel_name)
        );

    *out <<"\nepetraStateModel valid options:\n";
    epetraStateModel->getValidParameters()->print(
      *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
      );

    // C.2) Create the Thyra-wrapped ModelEvaluator

    RCP<Thyra::ModelEvaluator<double> > fwdStateModel =
      epetraModelEvaluator(epetraStateModel, W_factory);

    const RCP<const Thyra::VectorSpaceBase<Scalar> >
      x_space = fwdStateModel->get_x_space();

    const RCP<const Thyra::VectorBase<Scalar> >
      gamma = Thyra::create_Vector(epetraStateModel->get_gamma(), x_space);
    *out << "\ngamma = " << describe(*gamma, solnVerbLevel);

    //
    *out << "\nD) Create the stepper and integrator for the forward problem ...\n";
    //

    RCP<Rythmos::TimeStepNonlinearSolver<double> > fwdTimeStepSolver =
      Rythmos::timeStepNonlinearSolver<double>();
    RCP<Rythmos::StepperBase<Scalar> > fwdStateStepper =
      Rythmos::backwardEulerStepper<double>(fwdStateModel, fwdTimeStepSolver);
    fwdStateStepper->setParameterList(sublist(paramList, RythmosStepper_name));
    RCP<Rythmos::IntegratorBase<Scalar> > fwdStateIntegrator;
    {
      RCP<ParameterList>
        integrationControlPL = sublist(paramList, RythmosIntegrationControl_name);
      integrationControlPL->set( "Take Variable Steps", false );
      integrationControlPL->set( "Fixed dt", as<double>(delta_t) );
      RCP<Rythmos::IntegratorBase<Scalar> >
        defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(
          Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL)
          );
      fwdStateIntegrator = defaultIntegrator;
    }
    fwdStateIntegrator->setParameterList(sublist(paramList, RythmosIntegrator_name));

    //
    *out << "\nE) Solve the forward problem ...\n";
    //

    const MEB::InArgs<Scalar>
      state_ic = fwdStateModel->getNominalValues();
    *out << "\nstate_ic:\n" << describe(state_ic,solnVerbLevel);

    fwdStateStepper->setInitialCondition(state_ic);
    fwdStateIntegrator->setStepper(fwdStateStepper, t_final);

    Array<RCP<const Thyra::VectorBase<Scalar> > > x_final_array;
    fwdStateIntegrator->getFwdPoints(
      Teuchos::tuple<Scalar>(t_final), &x_final_array, NULL, NULL
      );
    const RCP<const Thyra::VectorBase<Scalar> > x_final = x_final_array[0];

    *out << "\nx_final:\n" << describe(*x_final, solnVerbLevel);

    //
    *out << "\nF) Check the solution to the forward problem ...\n";
    //

    const RCP<Thyra::VectorBase<Scalar> >
      x_beta = createMember(x_space),
      x_final_be_exact = createMember(x_space);

    {
      Thyra::ConstDetachedVectorView<Scalar> d_gamma(*gamma);
      Thyra::ConstDetachedVectorView<Scalar> d_x_ic(*state_ic.get_x());
      Thyra::DetachedVectorView<Scalar> d_x_beta(*x_beta);
      Thyra::DetachedVectorView<Scalar> d_x_final_be_exact(*x_final_be_exact);
      const int n = d_gamma.subDim();
      for ( int i = 0; i < n; ++i ) {
        d_x_beta(i) = 1.0 / ( 1.0 - delta_t * d_gamma(i) );
        d_x_final_be_exact(i) = integralPow(d_x_beta(i), numTimeSteps) * d_x_ic(i);
      }
    }

    *out << "\nx_final_be_exact:\n" << describe(*x_final_be_exact, solnVerbLevel);

    result = Thyra::testRelNormDiffErr<Scalar>(
      "x_final", *x_final,
      "x_final_be_exact", *x_final_be_exact,
      "maxStateError", maxStateError,
      "warningTol", 1.0, // Don't warn
      &*out, solnVerbLevel
      );
    if (!result) success = false;

    //
    *out << "\nG) Create the Adjoint ME wrapper object ...\n";
    //

    RCP<Thyra::ModelEvaluator<double> > adjModel =
      Rythmos::adjointModelEvaluator<double>(
        fwdStateModel, fwdTimeRange
        );

    //
    *out << "\nH) Create a stepper and integrator for the adjoint ...\n";
    //

    RCP<Thyra::LinearNonlinearSolver<double> > adjTimeStepSolver =
      Thyra::linearNonlinearSolver<double>();
    RCP<Rythmos::StepperBase<Scalar> > adjStepper =
      Rythmos::backwardEulerStepper<double>(adjModel, adjTimeStepSolver);
    adjStepper->setParameterList(sublist(paramList, RythmosStepper_name));
    RCP<Rythmos::IntegratorBase<Scalar> > adjIntegrator =
      fwdStateIntegrator->cloneIntegrator();

    //
    *out << "\nI) Set up the initial condition for the adjoint at the final time ...\n";
    //

    const RCP<const Thyra::VectorSpaceBase<Scalar> >
      f_space = fwdStateModel->get_f_space();

    // lambda(t_final) = x_final
    const RCP<Thyra::VectorBase<Scalar> > lambda_ic = createMember(f_space);
    V_V( outArg(*lambda_ic), *x_final_be_exact );

    // lambda_dot(t_final,i) = - gamma(i) * lambda(t_final,i)
    const RCP<Thyra::VectorBase<Scalar> > lambda_dot_ic = createMember(f_space);
    Thyra::V_S<Scalar>( outArg(*lambda_dot_ic), ST::zero() );
    Thyra::ele_wise_prod<Scalar>( -ST::one(), *gamma, *lambda_ic,
      outArg(*lambda_dot_ic) );

    MEB::InArgs<Scalar> adj_ic = adjModel->getNominalValues();
    adj_ic.set_x(lambda_ic);
    adj_ic.set_x_dot(lambda_dot_ic);
    *out << "\nadj_ic:\n" << describe(adj_ic,solnVerbLevel);

    //
    *out << "\nJ) Integrate the adjoint backwards in time (using backward time) ...\n";
    //

    adjStepper->setInitialCondition(adj_ic);
    adjIntegrator->setStepper(adjStepper, fwdTimeRange.length());

    Array<RCP<const Thyra::VectorBase<Scalar> > > lambda_final_array;
    adjIntegrator->getFwdPoints(
      Teuchos::tuple<Scalar>(fwdTimeRange.length()), &lambda_final_array, NULL, NULL
      );
    const RCP<const Thyra::VectorBase<Scalar> > lambda_final = lambda_final_array[0];

    *out << "\nlambda_final:\n" << describe(*lambda_final, solnVerbLevel);

    //
    *out << "\nK) Test the final adjoint againt exact discrete solution ...\n";
    //

    {

      const RCP<Thyra::VectorBase<Scalar> >
        lambda_final_be_exact = createMember(x_space);

      {
        Thyra::ConstDetachedVectorView<Scalar> d_gamma(*gamma);
        Thyra::ConstDetachedVectorView<Scalar> d_x_final(*x_final);
        Thyra::DetachedVectorView<Scalar> d_x_beta(*x_beta);
        Thyra::DetachedVectorView<Scalar> d_lambda_final_be_exact(*lambda_final_be_exact);
        const int n = d_gamma.subDim();
        for ( int i = 0; i < n; ++i ) {
          d_lambda_final_be_exact(i) = integralPow(d_x_beta(i), numTimeSteps) * d_x_final(i);
        }
      }

      *out << "\nlambda_final_be_exact:\n" << describe(*lambda_final_be_exact, solnVerbLevel);

      result = Thyra::testRelNormDiffErr<Scalar>(
        "lambda_final", *lambda_final,
        "lambda_final_be_exact", *lambda_final_be_exact,
        "maxStateError", maxStateError,
        "warningTol", 1.0, // Don't warn
        &*out, solnVerbLevel
        );
      if (!result) success = false;

    }

    //
    *out << "\nL) Test the reduced gradient from the adjoint against the discrete forward reduced gradient ...\n";
    //

    {

      const RCP<const Thyra::VectorBase<Scalar> >
        d_d_hat_d_p_from_lambda = lambda_final; // See above

      const RCP<Thyra::VectorBase<Scalar> >
        d_d_hat_d_p_be_exact = createMember(x_space);

      {
        Thyra::ConstDetachedVectorView<Scalar> d_x_ic(*state_ic.get_x());
        Thyra::DetachedVectorView<Scalar> d_x_beta(*x_beta);
        Thyra::DetachedVectorView<Scalar> d_d_d_hat_d_p_be_exact(*d_d_hat_d_p_be_exact);
        const int n = d_x_ic.subDim();
        for ( int i = 0; i < n; ++i ) {
          d_d_d_hat_d_p_be_exact(i) = integralPow(d_x_beta(i), 2*numTimeSteps) * d_x_ic(i);
        }
      }

      *out << "\nd_d_hat_d_p_be_exact:\n" << describe(*d_d_hat_d_p_be_exact, solnVerbLevel);

      result = Thyra::testRelNormDiffErr<Scalar>(
        "d_d_hat_d_p_from_lambda", *d_d_hat_d_p_from_lambda,
        "d_d_hat_d_p_be_exact", *d_d_hat_d_p_be_exact,
        "maxStateError", maxStateError,
        "warningTol", 1.0, // Don't warn
        &*out, solnVerbLevel
        );
      if (!result) success = false;

    }

  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(true,*out,success);

  if(success)
    *out << "\nEnd Result: TEST PASSED" << endl;
  else
    *out << "\nEnd Result: TEST FAILED" << endl;

  return ( success ? 0 : 1 );

} // end main() [Doxygen looks for this!]
Example #9
0
bool VectorSpaceTester<Scalar>::check(
  const VectorSpaceBase<Scalar>  &vs
  ,Teuchos::FancyOStream         *out_arg
  ) const
{
  using std::endl;
  using Teuchos::describe;
  using Teuchos::FancyOStream;
  using Teuchos::OSTab;
  typedef Teuchos::ScalarTraits<Scalar> ST;
  //typedef typename ST::magnitudeType    ScalarMag;

  Teuchos::RCP<FancyOStream> out = Teuchos::rcp(out_arg,false);
  const Teuchos::EVerbosityLevel verbLevel = (dump_all()?Teuchos::VERB_EXTREME:Teuchos::VERB_MEDIUM);

  OSTab tab(out,1,"THYRA");

  bool result, success = true;

  if(out.get()) *out <<endl<< "*** Entering Thyra::VectorSpaceTester<"<<ST::name()<<">::check(vs,...) ...\n";

  if(out.get()) *out <<endl<< "Testing a vector space vs described as:\n" << describe(vs,verbLevel);

  if(out.get())
    *out
      <<endl<< "A) Calling basic query functions ...\n"
      <<endl<< "vs.dim() = " << vs.dim()
      <<endl<< "vs.hasInCoreView() = " << vs.hasInCoreView() << std::endl;

  if(out.get()) *out <<endl<< "B) Checking that vs is compatible with itself ...\n";

  if(out.get()) *out <<endl<< "vs.isCompatible(vs)=";
  result = vs.isCompatible(vs);
  if(!result) success = false;
  if(out.get()) *out << result << " == true : " << passfail(result) << std::endl;

  if(out.get()) *out <<endl<< "C) Creating a randomized vector member v ...\n";
  Teuchos::RCP<Thyra::VectorBase<Scalar> >
    v = createMember(vs);
  randomize(Scalar(-ST::one()),Scalar(+ST::one()),v.ptr());

  if(out.get()) *out <<endl<< "D) Testing the VectorBase interface of v ...\n";

  result = vectorTester_.check(*v,out.get());
  if(!result) success = false;

  if(out.get()) *out <<endl<< "C) Creating a randomized MultiVector member mv ...\n";
  Teuchos::RCP<Thyra::MultiVectorBase<Scalar> >
    mv = createMembers(vs,num_mv_cols());
  randomize(Scalar(-ST::one()),Scalar(+ST::one()),mv.ptr());

  if(out.get()) *out <<endl<< "D) Testing the MultiVectorBase interface of mv ...\n";

  result = vectorTester_.multiVectorTester().check(*mv, out.ptr());
  if(!result) success = false;

  if(out.get()) {
    if(success)
      *out << endl <<"Congratulations, this VectorSpaceBase object seems to check out!\n";
    else
      *out << endl <<"Oh no, at least one of the tests performed with this VectorSpaceBase object failed (see above failures)!\n";
    *out << endl << "*** Leaving VectorSpaceTester<"<<ST::name()<<">::check(vs,...)\n";
  }
  
  return success;

}
int main(int argc, char *argv[])
{

  using std::endl;
  typedef double Scalar;
  typedef double ScalarMag;
  using Teuchos::describe;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::rcp_implicit_cast;
  using Teuchos::rcp_dynamic_cast;
  using Teuchos::as;
  using Teuchos::ParameterList;
  using Teuchos::CommandLineProcessor;
  typedef Teuchos::ParameterList::PrintOptions PLPrintOptions;
  typedef Thyra::ModelEvaluatorBase MEB;
  typedef Thyra::DefaultMultiVectorProductVectorSpace<Scalar> DMVPVS;
  using Thyra::productVectorBase;

  bool result, success = true;

  Teuchos::GlobalMPISession mpiSession(&argc,&argv);

  RCP<Epetra_Comm> epetra_comm;
#ifdef HAVE_MPI
  epetra_comm = rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
  epetra_comm = rcp( new Epetra_SerialComm );
#endif // HAVE_MPI

  RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

  try {

    //
    // Read commandline options
    //

    CommandLineProcessor clp;
    clp.throwExceptions(false);
    clp.addOutputSetupOptions(true);

    std::string paramsFileName = "";
    clp.setOption( "params-file", &paramsFileName,
      "File name for XML parameters" );

    std::string extraParamsString = "";
    clp.setOption( "extra-params", &extraParamsString,
      "Extra XML parameters" );

    std::string extraParamsFile = "";
    clp.setOption( "extra-params-file", &extraParamsFile, "File containing extra parameters in XML format.");

    double maxStateError = 1e-6;
    clp.setOption( "max-state-error", &maxStateError,
      "The maximum allowed error in the integrated state in relation to the exact state solution" );

    double finalTime = 1e-3;
    clp.setOption( "final-time", &finalTime,
      "Final integration time (initial time is 0.0)" );

    int numTimeSteps = 10;
    clp.setOption( "num-time-steps", &numTimeSteps,
      "Number of (fixed) time steps.  If <= 0.0, then variable time steps are taken" );

    bool useBDF = false;
    clp.setOption( "use-BDF", "use-BE", &useBDF,
      "Use BDF or Backward Euler (BE)" );

    bool useIRK = false;
    clp.setOption( "use-IRK", "use-other", &useIRK,
      "Use IRK or something" );

    bool doFwdSensSolve = false;
    clp.setOption( "fwd-sens-solve", "state-solve", &doFwdSensSolve,
      "Do the forward sensitivity solve or just the state solve" );

    bool doFwdSensErrorControl = false;
    clp.setOption( "fwd-sens-err-cntrl", "no-fwd-sens-err-cntrl", &doFwdSensErrorControl,
      "Do error control on the forward sensitivity solve or not" );

    double maxRestateError = 0.0;
    clp.setOption( "max-restate-error", &maxRestateError,
      "The maximum allowed error between the state integrated by itself verses integrated along with DxDp" );

    double maxSensError = 1e-4;
    clp.setOption( "max-sens-error", &maxSensError,
      "The maximum allowed error in the integrated sensitivity in relation to"
      " the finite-difference sensitivity" );

    Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_DEFAULT;
    setVerbosityLevelOption( "verb-level", &verbLevel,
      "Top-level verbosity level.  By default, this gets deincremented as you go deeper into numerical objects.",
      &clp );

    bool testExactSensitivity = false;
    clp.setOption( "test-exact-sens", "no-test-exact-sens", &testExactSensitivity,
      "Test the exact sensitivity with finite differences or not." );

    bool dumpFinalSolutions = false;
    clp.setOption(
      "dump-final-solutions", "no-dump-final-solutions", &dumpFinalSolutions,
      "Determine if the final solutions are dumpped or not." );

    CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
    if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;

    if ( Teuchos::VERB_DEFAULT == verbLevel )
      verbLevel = Teuchos::VERB_LOW;

    const Teuchos::EVerbosityLevel
      solnVerbLevel = ( dumpFinalSolutions ? Teuchos::VERB_EXTREME : verbLevel );

    //
    // Get the base parameter list that all other parameter lists will be read
    // from.
    //

    RCP<ParameterList>
      paramList = Teuchos::parameterList();
    if (paramsFileName.length())
      updateParametersFromXmlFile( paramsFileName, paramList.ptr() );
    if(extraParamsFile.length())
      Teuchos::updateParametersFromXmlFile( "./"+extraParamsFile, paramList.ptr() );
    if (extraParamsString.length())
      updateParametersFromXmlString( extraParamsString, paramList.ptr() );

    if (testExactSensitivity) {
      paramList->sublist(DiagonalTransientModel_name).set("Exact Solution as Response",true);
    }

    paramList->validateParameters(*getValidParameters(),0); // Only validate top level lists!

    //
    // Create the Stratimikos linear solver factory.
    //
    // This is the linear solve strategy that will be used to solve for the
    // linear system with the W.
    //

    Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
    linearSolverBuilder.setParameterList(sublist(paramList,Stratimikos_name));
    RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
      W_factory = createLinearSolveStrategy(linearSolverBuilder);

    //
    // Create the underlying EpetraExt::ModelEvaluator
    //

    RCP<EpetraExt::DiagonalTransientModel>
      epetraStateModel = EpetraExt::diagonalTransientModel(
        epetra_comm,
        sublist(paramList,DiagonalTransientModel_name)
        );

    *out <<"\nepetraStateModel valid options:\n";
    epetraStateModel->getValidParameters()->print(
      *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
      );

    //
    // Create the Thyra-wrapped ModelEvaluator
    //

    RCP<Thyra::ModelEvaluator<double> >
      stateModel = epetraModelEvaluator(epetraStateModel,W_factory);

    *out << "\nParameter names = " << *stateModel->get_p_names(0) << "\n";

    //
    // Create the Rythmos stateStepper
    //

    RCP<Rythmos::TimeStepNonlinearSolver<double> >
      nonlinearSolver = Rythmos::timeStepNonlinearSolver<double>();
    RCP<ParameterList>
      nonlinearSolverPL = sublist(paramList,TimeStepNonlinearSolver_name);
    nonlinearSolverPL->get("Default Tol",1e-3*maxStateError); // Set default if not set
    nonlinearSolver->setParameterList(nonlinearSolverPL);

    RCP<Rythmos::StepperBase<Scalar> > stateStepper;

    if (useBDF) {
      stateStepper = rcp(
        new Rythmos::ImplicitBDFStepper<double>(
          stateModel, nonlinearSolver
          )
        );
    }
    else if (useIRK) {
      // We need a separate LOWSFB object for the IRK stepper
      RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
        irk_W_factory = createLinearSolveStrategy(linearSolverBuilder);
      RCP<Rythmos::RKButcherTableauBase<double> > irkbt = Rythmos::createRKBT<double>("Backward Euler");
      stateStepper = Rythmos::implicitRKStepper<double>(
        stateModel, nonlinearSolver, irk_W_factory, irkbt
        );
    }
    else {
      stateStepper = rcp(
        new Rythmos::BackwardEulerStepper<double>(
          stateModel, nonlinearSolver
          )
        );
    }

    *out <<"\nstateStepper:\n" << describe(*stateStepper,verbLevel);
    *out <<"\nstateStepper valid options:\n";
    stateStepper->getValidParameters()->print(
      *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
      );

    stateStepper->setParameterList(sublist(paramList,RythmosStepper_name));

    //
    // Setup finite difference objects that will be used for tests
    //

    Thyra::DirectionalFiniteDiffCalculator<Scalar> fdCalc;
    fdCalc.setParameterList(sublist(paramList,FdCalc_name));
    fdCalc.setOStream(out);
    fdCalc.setVerbLevel(verbLevel);

    //
    // Use a StepperAsModelEvaluator to integrate the state
    //

    const MEB::InArgs<Scalar>
      state_ic = stateModel->getNominalValues();
    *out << "\nstate_ic:\n" << describe(state_ic,verbLevel);

    RCP<Rythmos::IntegratorBase<Scalar> > integrator;
    {
      RCP<ParameterList>
        integratorPL = sublist(paramList,RythmosIntegrator_name);
      integratorPL->set( "Take Variable Steps", as<bool>(numTimeSteps < 0) );
      integratorPL->set( "Fixed dt", as<double>((finalTime - state_ic.get_t())/numTimeSteps) );
      RCP<Rythmos::IntegratorBase<Scalar> >
        defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(
          Rythmos::simpleIntegrationControlStrategy<Scalar>(integratorPL)
          );
      integrator = defaultIntegrator;
    }

    RCP<Rythmos::StepperAsModelEvaluator<Scalar> >
      stateIntegratorAsModel = Rythmos::stepperAsModelEvaluator(
        stateStepper, integrator, state_ic
        );
    stateIntegratorAsModel->setVerbLevel(verbLevel);

    *out << "\nUse the StepperAsModelEvaluator to integrate state x(p,finalTime) ... \n";

    RCP<Thyra::VectorBase<Scalar> > x_final;

    {

      Teuchos::OSTab tab(out);

      x_final = createMember(stateIntegratorAsModel->get_g_space(0));

      eval_g(
        *stateIntegratorAsModel,
        0, *state_ic.get_p(0),
        finalTime,
        0, &*x_final
        );

      *out
        << "\nx_final = x(p,finalTime) evaluated using stateIntegratorAsModel:\n"
        << describe(*x_final,solnVerbLevel);

    }

    //
    // Test the integrated state against the exact analytical state solution
    //

    RCP<const Thyra::VectorBase<Scalar> >
      exact_x_final = create_Vector(
        epetraStateModel->getExactSolution(finalTime),
        stateModel->get_x_space()
        );

    result = Thyra::testRelNormDiffErr(
      "exact_x_final", *exact_x_final, "x_final", *x_final,
      "maxStateError", maxStateError, "warningTol", 1.0, // Don't warn
      &*out, solnVerbLevel
      );
    if (!result) success = false;

    //
    // Solve and test the forward sensitivity computation
    //

    if (doFwdSensSolve) {

      //
      // Create the forward sensitivity stepper
      //

      RCP<Rythmos::ForwardSensitivityStepper<Scalar> > stateAndSensStepper =
        Rythmos::forwardSensitivityStepper<Scalar>();
      if (doFwdSensErrorControl) {
        stateAndSensStepper->initializeDecoupledSteppers(
          stateModel, 0, stateModel->getNominalValues(),
          stateStepper, nonlinearSolver,
          integrator->cloneIntegrator(), finalTime
          );
      }
      else {
        stateAndSensStepper->initializeSyncedSteppers(
          stateModel, 0, stateModel->getNominalValues(),
          stateStepper, nonlinearSolver
          );
        // The above call will result in stateStepper and nonlinearSolver being
        // cloned.  This helps to ensure consistency between the state and
        // sensitivity computations!
      }

      //
      // Set the initial condition for the state and forward sensitivities
      //

      RCP<Thyra::VectorBase<Scalar> > s_bar_init
        = createMember(stateAndSensStepper->getFwdSensModel()->get_x_space());
      assign( s_bar_init.ptr(), 0.0 );
      RCP<Thyra::VectorBase<Scalar> > s_bar_dot_init
        = createMember(stateAndSensStepper->getFwdSensModel()->get_x_space());
      assign( s_bar_dot_init.ptr(), 0.0 );
      // Above, I believe that these are the correct initial conditions for
      // s_bar and s_bar_dot given how the EpetraExt::DiagonalTransientModel
      // is currently implemented!

      RCP<const Rythmos::StateAndForwardSensitivityModelEvaluator<Scalar> >
        stateAndSensModel = stateAndSensStepper->getStateAndFwdSensModel();

      MEB::InArgs<Scalar>
        state_and_sens_ic = stateAndSensStepper->getModel()->createInArgs();

      // Copy time, parameters etc.
      state_and_sens_ic.setArgs(state_ic);
      // Set initial condition for x_bar = [ x; s_bar ]
      state_and_sens_ic.set_x(
        stateAndSensModel->create_x_bar_vec(state_ic.get_x(),s_bar_init)
        );
      // Set initial condition for x_bar_dot = [ x_dot; s_bar_dot ]
      state_and_sens_ic.set_x_dot(
        stateAndSensModel->create_x_bar_vec(state_ic.get_x_dot(),s_bar_dot_init)
        );

      *out << "\nstate_and_sens_ic:\n" << describe(state_and_sens_ic,verbLevel);

      stateAndSensStepper->setInitialCondition(state_and_sens_ic);

      //
      // Use a StepperAsModelEvaluator to integrate the state+sens
      //

      RCP<Rythmos::StepperAsModelEvaluator<Scalar> >
        stateAndSensIntegratorAsModel = Rythmos::stepperAsModelEvaluator(
          rcp_implicit_cast<Rythmos::StepperBase<Scalar> >(stateAndSensStepper),
          integrator, state_and_sens_ic
          );
      stateAndSensIntegratorAsModel->setVerbLevel(verbLevel);

      *out << "\nUse the StepperAsModelEvaluator to integrate state + sens x_bar(p,finalTime) ... \n";

      RCP<Thyra::VectorBase<Scalar> > x_bar_final;

      {

        Teuchos::OSTab tab(out);

        x_bar_final = createMember(stateAndSensIntegratorAsModel->get_g_space(0));

        eval_g(
          *stateAndSensIntegratorAsModel,
          0, *state_ic.get_p(0),
          finalTime,
          0, &*x_bar_final
          );

        *out
          << "\nx_bar_final = x_bar(p,finalTime) evaluated using stateAndSensIntegratorAsModel:\n"
          << describe(*x_bar_final,solnVerbLevel);

      }

      //
      // Test that the state computed above is same as computed initially!
      //

      *out << "\nChecking that x(p,finalTime) computed as part of x_bar above is the same ...\n";

      {

        Teuchos::OSTab tab(out);

        RCP<const Thyra::VectorBase<Scalar> >
          x_in_x_bar_final = productVectorBase<Scalar>(x_bar_final)->getVectorBlock(0);

        result = Thyra::testRelNormDiffErr<Scalar>(
          "x_final", *x_final,
          "x_in_x_bar_final", *x_in_x_bar_final,
          "maxRestateError", maxRestateError,
          "warningTol", 1.0, // Don't warn
          &*out, solnVerbLevel
          );
        if (!result) success = false;

      }

      //
      // Compute DxDp using finite differences
      //

      *out << "\nApproximating DxDp(p,t) using directional finite differences of integrator for x(p,t) ...\n";

      RCP<Thyra::MultiVectorBase<Scalar> > DxDp_fd_final;

      {

        Teuchos::OSTab tab(out);


        MEB::InArgs<Scalar>
          fdBasePoint = stateIntegratorAsModel->createInArgs();

        fdBasePoint.set_t(finalTime);
        fdBasePoint.set_p(0,stateModel->getNominalValues().get_p(0));

        DxDp_fd_final = createMembers(
          stateIntegratorAsModel->get_g_space(0),
          stateIntegratorAsModel->get_p_space(0)->dim()
          );

        typedef Thyra::DirectionalFiniteDiffCalculatorTypes::SelectedDerivatives
          SelectedDerivatives;

        MEB::OutArgs<Scalar> fdOutArgs =
          fdCalc.createOutArgs(
            *stateIntegratorAsModel,
            SelectedDerivatives().supports(MEB::OUT_ARG_DgDp,0,0)
            );
        fdOutArgs.set_DgDp(0,0,DxDp_fd_final);

        // Silence the model evaluators that are called.  The fdCal object
        // will show all of the inputs and outputs for each call.
        stateStepper->setVerbLevel(Teuchos::VERB_NONE);
        stateIntegratorAsModel->setVerbLevel(Teuchos::VERB_NONE);

        fdCalc.calcDerivatives(
          *stateIntegratorAsModel, fdBasePoint,
          stateIntegratorAsModel->createOutArgs(), // Don't bother with function value
          fdOutArgs
          );

        *out
          << "\nFinite difference DxDp_fd_final = DxDp(p,finalTime): "
          << describe(*DxDp_fd_final,solnVerbLevel);

      }

      //
      // Test that the integrated sens and the F.D. sens are similar
      //

      *out << "\nChecking that integrated DxDp(p,finalTime) and finite-diff DxDp(p,finalTime) are similar ...\n";

      {

        Teuchos::OSTab tab(out);

        RCP<const Thyra::VectorBase<Scalar> >
          DxDp_vec_final = Thyra::productVectorBase<Scalar>(x_bar_final)->getVectorBlock(1);

        RCP<const Thyra::VectorBase<Scalar> >
          DxDp_fd_vec_final = Thyra::multiVectorProductVector(
            rcp_dynamic_cast<const Thyra::DefaultMultiVectorProductVectorSpace<Scalar> >(
              DxDp_vec_final->range()
              ),
            DxDp_fd_final
            );

        result = Thyra::testRelNormDiffErr(
          "DxDp_vec_final", *DxDp_vec_final,
          "DxDp_fd_vec_final", *DxDp_fd_vec_final,
          "maxSensError", maxSensError,
          "warningTol", 1.0, // Don't warn
          &*out, solnVerbLevel
          );
        if (!result) success = false;

      }

    }

  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(true,*out,success);

  if(success)
    *out << "\nEnd Result: TEST PASSED" << endl;
  else
    *out << "\nEnd Result: TEST FAILED" << endl;

  return ( success ? 0 : 1 );

} // end main() [Doxygen looks for this!]
void AztecOOLinearOpWithSolve::describe(
  Teuchos::FancyOStream &out,
  const Teuchos::EVerbosityLevel verbLevel
  ) const
{
  using Teuchos::OSTab;
  using Teuchos::typeName;
  using Teuchos::describe;
  switch(verbLevel) {
    case Teuchos::VERB_DEFAULT:
    case Teuchos::VERB_LOW:
      out << this->description() << std::endl;
      break;
    case Teuchos::VERB_MEDIUM:
    case Teuchos::VERB_HIGH:
    case Teuchos::VERB_EXTREME:
    {
      out
        << Teuchos::Describable::description() << "{"
        << "rangeDim=" << this->range()->dim()
        << ",domainDim="<< this->domain()->dim() << "}\n";
      OSTab tab(out);
      if (!is_null(fwdOp_)) {
        out << "fwdOp = " << describe(*fwdOp_,verbLevel);
      }
      if (!is_null(prec_)) {
        out << "prec = " << describe(*prec_,verbLevel);
      }
      if (!is_null(aztecFwdSolver_)) {
        if (aztecFwdSolver_->GetUserOperator())
          out
            << "Aztec Fwd Op = "
            << typeName(*aztecFwdSolver_->GetUserOperator()) << "\n";
        if (aztecFwdSolver_->GetUserMatrix())
          out
            << "Aztec Fwd Mat = "
            << typeName(*aztecFwdSolver_->GetUserMatrix()) << "\n";
        if (aztecFwdSolver_->GetPrecOperator())
          out
            << "Aztec Fwd Prec Op = "
            << typeName(*aztecFwdSolver_->GetPrecOperator()) << "\n";
        if (aztecFwdSolver_->GetPrecMatrix())
          out
            << "Aztec Fwd Prec Mat = "
            << typeName(*aztecFwdSolver_->GetPrecMatrix()) << "\n";
      }
      if (!is_null(aztecAdjSolver_)) {
        if (aztecAdjSolver_->GetUserOperator())
          out
            << "Aztec Adj Op = "
            << typeName(*aztecAdjSolver_->GetUserOperator()) << "\n";
        if (aztecAdjSolver_->GetUserMatrix())
          out
            << "Aztec Adj Mat = "
            << typeName(*aztecAdjSolver_->GetUserMatrix()) << "\n";
        if (aztecAdjSolver_->GetPrecOperator())
          out
            << "Aztec Adj Prec Op = "
            << typeName(*aztecAdjSolver_->GetPrecOperator()) << "\n";
        if (aztecAdjSolver_->GetPrecMatrix())
          out
            << "Aztec Adj Prec Mat = "
            << typeName(*aztecAdjSolver_->GetPrecMatrix()) << "\n";
      }
      break;
    }
    default:
      TEUCHOS_TEST_FOR_EXCEPT(true); // Should never get here!
  }
}
int main(int argc, char* argv[])
{

    using Teuchos::describe;
    using Teuchos::rcp;
    using Teuchos::rcp_dynamic_cast;
    using Teuchos::rcp_const_cast;
    using Teuchos::RCP;
    using Teuchos::CommandLineProcessor;
    using Teuchos::ParameterList;
    using Teuchos::sublist;
    using Teuchos::getParametersFromXmlFile;
    typedef ParameterList::PrintOptions PLPrintOptions;
    using Thyra::inverse;
    using Thyra::initializePreconditionedOp;
    using Thyra::initializeOp;
    using Thyra::unspecifiedPrec;
    using Thyra::solve;
    typedef RCP<const Thyra::LinearOpBase<double> > LinearOpPtr;
    typedef RCP<Thyra::VectorBase<double> > VectorPtr;

    bool success = true;
    bool verbose = true;

    Teuchos::GlobalMPISession mpiSession(&argc,&argv);

    Teuchos::RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

    try {

        //
        // Read in options from the command line
        //

        CommandLineProcessor clp(false); // Don't throw exceptions

        const int numVerbLevels = 6;
        Teuchos::EVerbosityLevel
        verbLevelValues[] =
        {
            Teuchos::VERB_DEFAULT, Teuchos::VERB_NONE,
            Teuchos::VERB_LOW, Teuchos::VERB_MEDIUM,
            Teuchos::VERB_HIGH, Teuchos::VERB_EXTREME
        };
        const char*
        verbLevelNames[] =
        { "default", "none", "low", "medium", "high", "extreme" };

        Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_MEDIUM;
        clp.setOption( "verb-level", &verbLevel,
                       numVerbLevels, verbLevelValues, verbLevelNames,
                       "Verbosity level used for all objects."
                     );

        std::string matrixFile = ".";
        clp.setOption( "matrix-file", &matrixFile,
                       "Matrix file."
                     );

        std::string paramListFile = "";
        clp.setOption( "param-list-file", &paramListFile,
                       "Parameter list for preconditioner and solver blocks."
                     );

        bool showParams = false;
        clp.setOption( "show-params", "no-show-params", &showParams,
                       "Show the parameter list or not."
                     );

        bool testPrecIsLinearOp = true;
        clp.setOption( "test-prec-is-linear-op", "test-prec-is-linear-op", &testPrecIsLinearOp,
                       "Test if the preconditioner is a linear operator or not."
                     );

        double solveTol = 1e-8;
        clp.setOption( "solve-tol", &solveTol,
                       "Tolerance for the solution to determine success or failure!"
                     );

        clp.setDocString(
            "This example program shows how to use one linear solver (e.g. AztecOO)\n"
            "as a preconditioner for another iterative solver (e.g. Belos).\n"
        );

        // Note: Use --help on the command line to see the above documentation

        CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
        if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;


        //
        *out << "\nA) Reading in the matrix ...\n";
        //

#ifdef HAVE_MPI
        Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
        Epetra_SerialComm comm;
#endif

        const LinearOpPtr A = readEpetraCrsMatrixFromMatrixMarketAsLinearOp(
                                  matrixFile, comm, "A");
        *out << "\nA = " << describe(*A,verbLevel) << "\n";

        const RCP<ParameterList> paramList = getParametersFromXmlFile(paramListFile);
        if (showParams) {
            *out << "\nRead in parameter list:\n\n";
            paramList->print(*out, PLPrintOptions().indent(2).showTypes(true));
        }

        //
        *out << "\nB) Get the preconditioner as a forward solver\n";
        //

        const RCP<ParameterList> precParamList = sublist(paramList, "Preconditioner Solver");
        Stratimikos::DefaultLinearSolverBuilder precSolverBuilder;
        precSolverBuilder.setParameterList(precParamList);
        const RCP<const Thyra::LinearOpWithSolveFactoryBase<double> > precSolverStrategy
            = createLinearSolveStrategy(precSolverBuilder);
        //precSolverStrategy->setVerbLevel(verbLevel);

        const LinearOpPtr A_inv_prec = inverse<double>(*precSolverStrategy, A,
                                       Thyra::SUPPORT_SOLVE_FORWARD_ONLY,
                                       Teuchos::null, // Use internal solve criteria
                                       Thyra::IGNORE_SOLVE_FAILURE // Ignore solve failures since this is just a prec
                                                      );
        *out << "\nA_inv_prec = " << describe(*A_inv_prec, verbLevel) << "\n";

        if (testPrecIsLinearOp) {
            *out << "\nTest that the preconditioner A_inv_prec is indeed a linear operator.\n";
            Thyra::LinearOpTester<double> linearOpTester;
            linearOpTester.check_adjoint(false);
            const bool linearOpCheck = linearOpTester.check(*A_inv_prec, out.ptr());
            if (!linearOpCheck) {
                success = false;
            }
        }

        //
        *out << "\nC) Create the forward solver using the created preconditioner ...\n";
        //

        const RCP<ParameterList> fwdSolverParamList = sublist(paramList, "Forward Solver");
        Stratimikos::DefaultLinearSolverBuilder fwdSolverSolverBuilder;
        fwdSolverSolverBuilder.setParameterList(fwdSolverParamList);
        const RCP<const Thyra::LinearOpWithSolveFactoryBase<double> > fwdSolverSolverStrategy
            = createLinearSolveStrategy(fwdSolverSolverBuilder);

        const RCP<Thyra::LinearOpWithSolveBase<double> >
        A_lows = fwdSolverSolverStrategy->createOp();

        initializePreconditionedOp<double>( *fwdSolverSolverStrategy, A,
                                            unspecifiedPrec(A_inv_prec), A_lows.ptr());
        //A_lows->setVerbLevel(verbLevel);
        *out << "\nA_lows = " << describe(*A_lows, verbLevel) << "\n";

        //
        *out << "\nD) Solve the linear system for a random RHS ...\n";
        //

        VectorPtr x = createMember(A->domain());
        VectorPtr b = createMember(A->range());
        Thyra::randomize(-1.0, +1.0, b.ptr());
        Thyra::assign(x.ptr(), 0.0); // Must give an initial guess!

        Thyra::SolveStatus<double>
        solveStatus = solve<double>( *A_lows, Thyra::NOTRANS, *b, x.ptr() );

        *out << "\nSolve status:\n" << solveStatus;

        *out << "\nSolution ||x|| = " << Thyra::norm(*x) << "\n";

        if(showParams) {
            *out << "\nParameter list after use:\n\n";
            paramList->print(*out, PLPrintOptions().indent(2).showTypes(true));
        }

        //
        *out << "\nF) Checking the error in the solution of r=b-A*x ...\n";
        //

        VectorPtr Ax = Thyra::createMember(b->space());
        Thyra::apply( *A, Thyra::NOTRANS, *x, Ax.ptr() );
        VectorPtr r = Thyra::createMember(b->space());
        Thyra::V_VmV<double>(r.ptr(), *b, *Ax);

        double
        Ax_nrm = Thyra::norm(*Ax),
        r_nrm = Thyra::norm(*r),
        b_nrm = Thyra::norm(*b),
        r_nrm_over_b_nrm = r_nrm / b_nrm;

        bool resid_tol_check = ( r_nrm_over_b_nrm <= solveTol );
        if(!resid_tol_check) success = false;

        *out
                << "\n||A*x|| = " << Ax_nrm << "\n";

        *out
                << "\n||A*x-b||/||b|| = " << r_nrm << "/" << b_nrm
                << " = " << r_nrm_over_b_nrm << " <= " << solveTol
                << " : " << Thyra::passfail(resid_tol_check) << "\n";

        Teuchos::TimeMonitor::summarize(*out<<"\n");
    }
    TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success)

    if (verbose) {
        if(success)  *out << "\nCongratulations! All of the tests checked out!\n";
        else         *out << "\nOh no! At least one of the tests failed!\n";
    }

    return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main(int argc, char *argv[])
{

    using std::endl;
    typedef double Scalar;
    typedef double ScalarMag;
    using Teuchos::describe;
    using Teuchos::RCP;
    using Teuchos::rcp;
    using Teuchos::rcp_implicit_cast;
    using Teuchos::rcp_dynamic_cast;
    using Teuchos::as;
    using Teuchos::ParameterList;
    using Teuchos::CommandLineProcessor;
    typedef Teuchos::ParameterList::PrintOptions PLPrintOptions;
    typedef Thyra::ModelEvaluatorBase MEB;
    using Thyra::createMember;
    using Thyra::createMembers;

    bool success = true;

    Teuchos::GlobalMPISession mpiSession(&argc,&argv);

    RCP<Epetra_Comm> epetra_comm;
#ifdef HAVE_MPI
    epetra_comm = rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
    epetra_comm = rcp( new Epetra_SerialComm );
#endif // HAVE_MPI

    RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

    try {

        //
        // A) Read commandline options
        //

        CommandLineProcessor clp;
        clp.throwExceptions(false);
        clp.addOutputSetupOptions(true);

        std::string paramsFileName = "";
        clp.setOption( "params-file", &paramsFileName,
                       "File name for XML parameters" );

        std::string extraParamsString = "";
        clp.setOption( "extra-params", &extraParamsString,
                       "Extra XML parameter string" );

        Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_DEFAULT;
        setVerbosityLevelOption( "verb-level", &verbLevel,
                                 "Top-level verbosity level.  By default, this gets deincremented as you go deeper into numerical objects.",
                                 &clp );

        double finalTime = 1.0;
        clp.setOption( "final-time", &finalTime, "Final time (the inital time)" );

        int numTimeSteps = 2;
        clp.setOption( "num-time-steps", &numTimeSteps, "Number of time steps" );

        bool dumpFinalSolutions = false;
        clp.setOption(
            "dump-final-solutions", "no-dump-final-solutions", &dumpFinalSolutions,
            "Determine if the final solutions are dumpped or not." );

        double maxStateError = 1e-6;
        clp.setOption( "max-state-error", &maxStateError,
                       "The maximum allowed error in the integrated state in relation to the exact state solution" );

        // ToDo: Read in more parameters

        CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
        if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;

        if ( Teuchos::VERB_DEFAULT == verbLevel )
            verbLevel = Teuchos::VERB_LOW;

        const Teuchos::EVerbosityLevel
        solnVerbLevel = ( dumpFinalSolutions ? Teuchos::VERB_EXTREME : verbLevel );

        //
        // B) Get the base parameter list that all other parameter lists will be
        // read from.
        //

        RCP<ParameterList> paramList = Teuchos::parameterList();
        if (paramsFileName.length())
            updateParametersFromXmlFile( paramsFileName, &*paramList );
        if (extraParamsString.length())
            updateParametersFromXmlString( extraParamsString, &*paramList );

        paramList->validateParameters(*getValidParameters());

        //
        // C) Create the Stratimikos linear solver factories.
        //

        // Get the linear solve strategy that will be used to solve for the linear
        // system with the dae's W matrix.
        Stratimikos::DefaultLinearSolverBuilder daeLinearSolverBuilder;
        daeLinearSolverBuilder.setParameterList(sublist(paramList,DAELinearSolver_name));
        RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
        daeLOWSF = createLinearSolveStrategy(daeLinearSolverBuilder);

        // Get the linear solve strategy that can be used to override the overall
        // linear system solve
        Stratimikos::DefaultLinearSolverBuilder overallLinearSolverBuilder;
        overallLinearSolverBuilder.setParameterList(sublist(paramList,OverallLinearSolver_name));
        RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
        overallLOWSF = createLinearSolveStrategy(overallLinearSolverBuilder);

        //
        // D) Create the underlying EpetraExt::ModelEvaluator
        //

        RCP<EpetraExt::DiagonalTransientModel> epetraDaeModel =
            EpetraExt::diagonalTransientModel(
                epetra_comm,
                sublist(paramList,DiagonalTransientModel_name)
            );

        *out <<"\nepetraDaeModel valid options:\n";
        epetraDaeModel->getValidParameters()->print(
            *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
        );

        //
        // E) Create the Thyra-wrapped ModelEvaluator
        //

        RCP<Thyra::ModelEvaluator<double> > daeModel =
            epetraModelEvaluator(epetraDaeModel,daeLOWSF);

        //
        // F) Create the TimeDiscretizedBackwardEulerModelEvaluator
        //

        MEB::InArgs<Scalar> initCond = daeModel->createInArgs();
        initCond.setArgs(daeModel->getNominalValues());

        RCP<Thyra::ModelEvaluator<Scalar> >
        discretizedModel = Rythmos::timeDiscretizedBackwardEulerModelEvaluator<Scalar>(
                               daeModel, initCond, finalTime, numTimeSteps, overallLOWSF );

        *out << "\ndiscretizedModel = " << describe(*discretizedModel,verbLevel);

        //
        // F) Setup a nonlinear solver and solve the system
        //

        // F.1) Setup a nonlinear solver

        Thyra::DampenedNewtonNonlinearSolver<Scalar> nonlinearSolver;
        nonlinearSolver.setOStream(out);
        nonlinearSolver.setVerbLevel(verbLevel);
        //nonlinearSolver.setParameterList(sublist(paramList,NonlinearSolver_name));
        //2007/11/27: rabartl: ToDo: Implement parameter list handling for
        //DampenedNonlinearSolve so that I can uncomment the above line.
        nonlinearSolver.setModel(discretizedModel);

        // F.2) Solve the system

        RCP<Thyra::VectorBase<Scalar> >
        x_bar = createMember(discretizedModel->get_x_space());
        V_S( x_bar.ptr(), 0.0 );

        Thyra::SolveStatus<Scalar> solveStatus =
            Thyra::solve( nonlinearSolver, &*x_bar );

        *out << "\nsolveStatus:\n" << solveStatus;

        *out << "\nx_bar = " << describe(*x_bar,solnVerbLevel);

        //
        // G) Verify that the solution is correct???
        //

        // Check against the end time exact solution.

        RCP<const Thyra::VectorBase<Scalar> >
        exact_x_final = Thyra::create_Vector(
                            epetraDaeModel->getExactSolution(finalTime),
                            daeModel->get_x_space()
                        );

        RCP<const Thyra::VectorBase<Scalar> > solved_x_final
            = rcp_dynamic_cast<Thyra::ProductVectorBase<Scalar> >(x_bar,true)->getVectorBlock(numTimeSteps-1);

        const bool result = Thyra::testRelNormDiffErr(
                                "exact_x_final", *exact_x_final, "solved_x_final", *solved_x_final,
                                "maxStateError", maxStateError, "warningTol", 1.0, // Don't warn
                                &*out, solnVerbLevel
                            );
        if (!result) success = false;

    }
    TEUCHOS_STANDARD_CATCH_STATEMENTS(true,*out,success);

    if(success)
        *out << "\nEnd Result: TEST PASSED" << endl;
    else
        *out << "\nEnd Result: TEST FAILED" << endl;

    return ( success ? 0 : 1 );

} // end main() [Doxygen looks for this!]
TEUCHOS_UNIT_TEST( EpetraLinearOp, blocked_op )
{

  if (Teuchos::GlobalMPISession::getNProc() > 2) {
    out << "Skipping test if numProc > 2 since it fails for some reason\n";
    return;
  }

  using Teuchos::describe;

  // build sub operators
  RCP<const LinearOpBase<double> > A00 =
    epetraLinearOp(getEpetraMatrix(4,4,0));
  RCP<const LinearOpBase<double> > A01 =
    epetraLinearOp(getEpetraMatrix(4,3,1));
  RCP<const LinearOpBase<double> > A02 =
    epetraLinearOp(getEpetraMatrix(4,2,2));
  RCP<const LinearOpBase<double> > A10 =
    epetraLinearOp(getEpetraMatrix(3,4,3));
  RCP<const LinearOpBase<double> > A11 =
    epetraLinearOp(getEpetraMatrix(3,3,4));
  RCP<const LinearOpBase<double> > A12 =
    epetraLinearOp(getEpetraMatrix(3,2,5));
  RCP<const LinearOpBase<double> > A20 =
    epetraLinearOp(getEpetraMatrix(2,4,6));
  RCP<const LinearOpBase<double> > A21 =
    epetraLinearOp(getEpetraMatrix(2,3,8));
  RCP<const LinearOpBase<double> > A22 =
    epetraLinearOp(getEpetraMatrix(2,2,8));

  const Teuchos::EVerbosityLevel verbLevel =
    (g_dumpAll ? Teuchos::VERB_HIGH : Teuchos::VERB_MEDIUM);

  out << "Sub operators built" << std::endl;

  {
     // build composite operator
     RCP<const LinearOpBase<double> > A =
       block2x2<double>(
         block2x2<double>(A00, A01, A10, A11),   block2x1<double>(A02,A12),
         block1x2<double>(A20, A21),             A22
         );

     out << "First composite operator built" << std::endl;

     // build vectors for use in apply
     RCP<MultiVectorBase<double> > x = createMembers<double>(A->domain(), 3);
     RCP<MultiVectorBase<double> > y = createMembers<double>(A->range(), 3);

     randomize(-1.0, 1.0, x.ptr());

     out << "A = \n" << describe(*A, verbLevel) << std::endl;
     out << "x = \n" << describe(*x, verbLevel) << std::endl;
     out << "y = \n" << describe(*y, verbLevel) << std::endl;

     // perform a matrix vector multiply
     apply(*A, NOTRANS, *x, y.ptr());

     out << "First composite operator completed" << std::endl;
  }

  {
     RCP<const LinearOpBase<double> > A = block2x2<double>(
       A11,                          block1x2<double>(A10, A12),
       block2x1<double>(A01, A21),   block2x2<double>(A00, A02, A20, A22)
       );

     out << "Second composite operator built" << std::endl;

     // build vectors for use in apply
     RCP<MultiVectorBase<double> > x = createMembers<double>(A->domain(), 3);
     RCP<MultiVectorBase<double> > y = createMembers<double>(A->range(), 3);

     randomize(-1.0, 1.0, x.ptr());

     out << "A = \n" << describe(*A, verbLevel) << std::endl;
     out << "x = \n" << describe(*x, verbLevel) << std::endl;
     out << "y = \n" << describe(*y, verbLevel) << std::endl;

     // perform a matrix vector multiply
     apply(*A, NOTRANS, *x, y.ptr());

     out << "Second composite operator completed" << std::endl;
  }

  out << "Test complete" << std::endl;

}