void TrilinosCrsMatrix::solve(LSS::Vector& solution, LSS::Vector& rhs)
{
  cf3_assert(m_is_created);
  cf3_assert(solution.is_created());
  cf3_assert(rhs.is_created());

  LSS::TrilinosVector& tsol = dynamic_cast<LSS::TrilinosVector&>(solution);
  LSS::TrilinosVector& trhs = dynamic_cast<LSS::TrilinosVector&>(rhs);

  Teuchos::RCP<Teuchos::ParameterList> paramList = Teuchos::getParametersFromXmlFile(options().option("settings_file").value_str());

  // Build Thyra linear algebra objects
  Teuchos::RCP<const Thyra::LinearOpBase<double> > th_mat = Thyra::epetraLinearOp(m_mat);
  Teuchos::RCP<const Thyra::VectorBase<double> > th_rhs = Thyra::create_Vector(trhs.epetra_vector(),th_mat->range());
  Teuchos::RCP<Thyra::VectorBase<double> > th_sol = Thyra::create_Vector(tsol.epetra_vector(),th_mat->domain());

  // Build stratimikos solver
  /////////////////////////////////////////////////////////

  Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;

  //Teko::addTekoToStratimikosBuilder(linearSolverBuilder);
  linearSolverBuilder.setParameterList(paramList);

  Teuchos::RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory = Thyra::createLinearSolveStrategy(linearSolverBuilder);
  Teuchos::RCP<Thyra::LinearOpWithSolveBase<double> > th_invA = Thyra::linearOpWithSolve(*lowsFactory, th_mat);

  Thyra::assign(th_sol.ptr(), 0.0);
  Thyra::SolveStatus<double> status = Thyra::solve<double>(*th_invA, Thyra::NOTRANS, *th_rhs, th_sol.ptr());
  CFinfo << "Thyra::solve finished with status " << status.message << CFendl;
}
Beispiel #2
0
//***********************************************************************
void NOX::Epetra::LinearSystemStratimikos::
initializeStratimikos(Teuchos::ParameterList& stratParams)
{
  Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;

#ifdef HAVE_NOX_TEKO
  Teko::addTekoToStratimikosBuilder(linearSolverBuilder);
#endif

  linearSolverBuilder.setParameterList(Teuchos::rcp(&stratParams, false));

  // Create a linear solver factory given information read from the
  // parameter list.
  lowsFactory = linearSolverBuilder.createLinearSolveStrategy("");

  // Setup output stream and the verbosity level
  lowsFactory->setOStream(outputStream);
  lowsFactory->setVerbLevel(Teuchos::VERB_LOW);

  // Initialize the LinearOpWithSolve
  lows = lowsFactory->createOp();

/*
  // Store sublist and std::string name where the Tolerance is set
  // so that inexact Newton can modify it in resetTolerance()
  if ("Belos" == stratParams.get("Linear Solver Type","Belos")) {
  }
*/
  

}
Beispiel #3
0
  virtual
  Teuchos::RCP<const Thyra::LinearOpWithSolveFactoryBase<double>>
  get_W_factory() const
  {
    Stratimikos::DefaultLinearSolverBuilder builder;

    const std::map<std::string, boost::any> linear_solver_params = {
      {"package", std::string("Belos")},
      {"method", std::string("Pseudo Block CG")},
      {"parameters", dict{
        {"Output Frequency", 1},
        {"Verbosity", 0}
      }}
      };
    auto p = Teuchos::rcp(new Teuchos::ParameterList());
    mikado::std_map_to_teuchos_list(
        mikado::convert_to_belos_parameters(linear_solver_params),
        *p
        );
    builder.setParameterList(p);

    auto lowsFactory = builder.createLinearSolveStrategy("");

    lowsFactory->setVerbLevel(Teuchos::VERB_LOW);

    return lowsFactory;
  }
TEUCHOS_UNIT_TEST(tStratimikosFactory, test_RelatedFunctions) 
{
  using Teuchos::RCP;
  using Teuchos::ParameterList;

  // build global (or serial communicator)
  #ifdef HAVE_MPI
     Epetra_MpiComm comm(MPI_COMM_WORLD);
  #else
     Epetra_SerialComm comm;
  #endif

  // build epetra operator
  RCP<Epetra_Operator> eA = buildStridedSystem(comm,5);
  RCP<Thyra::LinearOpBase<double> > tA = Thyra::nonconstEpetraLinearOp(eA);

  {
     // build stratimikos factory, adding Teko's version
     Stratimikos::DefaultLinearSolverBuilder stratFactory;
     
     Teko::addTekoToStratimikosBuilder(stratFactory);
     TEST_THROW(Teko::addTekoToStratimikosBuilder(stratFactory),std::logic_error);
     Teko::addTekoToStratimikosBuilder(stratFactory,"Teko-2");

     TEST_NOTHROW(stratFactory.getValidParameters()->sublist("Preconditioner Types").sublist("Teko"));
     TEST_NOTHROW(stratFactory.getValidParameters()->sublist("Preconditioner Types").sublist("Teko-2"));
  }

  {
     Teuchos::RCP<Teko::RequestHandler> rh = Teuchos::rcp(new Teko::RequestHandler);

     // build stratimikos factory, adding Teko's version
     Stratimikos::DefaultLinearSolverBuilder stratFactory;
     
     Teko::addTekoToStratimikosBuilder(stratFactory,rh);
     TEST_THROW(Teko::addTekoToStratimikosBuilder(stratFactory,rh),std::logic_error);
     Teko::addTekoToStratimikosBuilder(stratFactory,rh,"Teko-2");

     TEST_NOTHROW(stratFactory.getValidParameters()->sublist("Preconditioner Types").sublist("Teko"));
     TEST_NOTHROW(stratFactory.getValidParameters()->sublist("Preconditioner Types").sublist("Teko-2"));

     RCP<ParameterList> params = Teuchos::rcp(new ParameterList(*stratFactory.getValidParameters()));
     ParameterList & tekoList = params->sublist("Preconditioner Types").sublist("Teko");
     tekoList.set("Write Block Operator", false);
     tekoList.set("Test Block Operator", false);
     tekoList.set("Strided Blocking","1 1");
     tekoList.set("Inverse Type","BGS");
     ParameterList & ifl = tekoList.sublist("Inverse Factory Library");
     ifl.sublist("BGS").set("Type","Block Gauss-Seidel");
     ifl.sublist("BGS").set("Inverse Type","Amesos");
     stratFactory.setParameterList(params);

     RCP<Teko::StratimikosFactory> precFactory
           = Teuchos::rcp_dynamic_cast<Teko::StratimikosFactory>(stratFactory.createPreconditioningStrategy("Teko-2"));
     TEST_EQUALITY(precFactory->getRequestHandler(),rh);
  }
}
Beispiel #5
0
int main(int argc,char * argv[])
{
   // calls MPI_Init and MPI_Finalize
   Teuchos::GlobalMPISession mpiSession(&argc,&argv);

   // read in parameter list
   Teuchos::RCP<Teuchos::ParameterList> paramList = Teuchos::getParametersFromXmlFile("strat_example.xml"); 

   // build global communicator
#ifdef HAVE_MPI
   Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
   Epetra_SerialComm Comm;
#endif

   // Read in the matrix, store pointer as an RCP
   Epetra_CrsMatrix * ptrA = 0;
   EpetraExt::MatrixMarketFileToCrsMatrix("../data/nsjac.mm",Comm,ptrA);
   RCP<Epetra_CrsMatrix> A = rcp(ptrA);

   // read in the RHS vector
   Epetra_Vector * ptrb = 0;
   EpetraExt::MatrixMarketFileToVector("../data/nsrhs_test.mm",A->OperatorRangeMap(),ptrb);
   RCP<const Epetra_Vector> b = rcp(ptrb);

   // allocate vectors
   RCP<Epetra_Vector> x = rcp(new Epetra_Vector(A->OperatorDomainMap()));
   x->PutScalar(0.0);

   // Build Thyra linear algebra objects
   RCP<const Thyra::LinearOpBase<double> > th_A = Thyra::epetraLinearOp(A);
   RCP<const Thyra::VectorBase<double> > th_b = Thyra::create_Vector(b,th_A->range());
   RCP<Thyra::VectorBase<double> > th_x = Thyra::create_Vector(x,th_A->domain());

   // Build stratimikos solver
   /////////////////////////////////////////////////////////

   Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;

   Teko::addTekoToStratimikosBuilder(linearSolverBuilder);
   linearSolverBuilder.setParameterList(paramList);

   RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory 
       = Thyra::createLinearSolveStrategy(linearSolverBuilder);

   Teuchos::RCP<Thyra::LinearOpWithSolveBase<double> > th_invA 
      = Thyra::linearOpWithSolve(*lowsFactory, th_A);

   Thyra::assign(th_x.ptr(), 0.0);
   Thyra::SolveStatus<double> status 
       = Thyra::solve<double>(*th_invA, Thyra::NOTRANS, *th_b, th_x.ptr());

   return 0;
}
TEUCHOS_UNIT_TEST(tStratimikosFactory, test_Defaults) 
{
  using Teuchos::RCP;
  using Teuchos::ParameterList;

  // build global (or serial communicator)
  #ifdef HAVE_MPI
     Epetra_MpiComm comm(MPI_COMM_WORLD);
  #else
     Epetra_SerialComm comm;
  #endif

  // build epetra operator
  RCP<Epetra_Operator> eA = buildSystem(comm,5);
  RCP<Thyra::LinearOpBase<double> > tA = Thyra::nonconstEpetraLinearOp(eA);

  // build stratimikos factory, adding Teko's version
  Stratimikos::DefaultLinearSolverBuilder stratFactory;
  stratFactory.setPreconditioningStrategyFactory(
        Teuchos::abstractFactoryStd<Thyra::PreconditionerFactoryBase<double>,Teko::StratimikosFactory>(),
        "Teko");
  RCP<const ParameterList> validParams = stratFactory.getValidParameters();
  stratFactory.setParameterList(Teuchos::rcp(new Teuchos::ParameterList(*validParams)));

  // print out Teko's parameter list and fail if it doesn't exist!
  TEST_NOTHROW(validParams->sublist("Preconditioner Types").sublist("Teko").print(out,
        ParameterList::PrintOptions().showDoc(true).indent(2).showTypes(true)));

  // build teko preconditioner factory
  RCP<Thyra::PreconditionerFactoryBase<double> > precFactory
        = stratFactory.createPreconditioningStrategy("Teko");

  // make sure factory is built
  TEST_ASSERT(precFactory!=Teuchos::null);

  // build preconditioner
  RCP<Thyra::PreconditionerBase<double> > prec = Thyra::prec<double>(*precFactory,tA);
  TEST_ASSERT(prec!=Teuchos::null);

  // build an operator to test against
  RCP<const Teko::InverseLibrary> invLib = Teko::InverseLibrary::buildFromStratimikos();
  RCP<const Teko::InverseFactory> invFact = invLib->getInverseFactory("Amesos");
  Teko::LinearOp testOp = Teko::buildInverse(*invFact,tA);

  Teko::LinearOp precOp = prec->getUnspecifiedPrecOp();
  TEST_ASSERT(precOp!=Teuchos::null);

  Thyra::LinearOpTester<double> tester;
  tester.show_all_tests(true);
  tester.set_all_error_tol(0);
  TEST_ASSERT(tester.compare(*precOp,*testOp,Teuchos::ptrFromRef(out)));
}
Beispiel #7
0
TEUCHOS_UNIT_TEST(belos_gcrodr, multiple_solves)
{
   // build global (or serial communicator)
   #ifdef HAVE_MPI
      Epetra_MpiComm Comm(MPI_COMM_WORLD);
   #else
      Epetra_SerialComm Comm;
   #endif

   // build and allocate linear system
   Teuchos::RCP<Epetra_CrsMatrix> mat = buildMatrix(100,Comm);
   Teuchos::RCP<Epetra_Vector> x0 = rcp(new Epetra_Vector(mat->OperatorDomainMap()));
   Teuchos::RCP<Epetra_Vector> x1 = rcp(new Epetra_Vector(mat->OperatorDomainMap()));
   Teuchos::RCP<Epetra_Vector> b = rcp(new Epetra_Vector(mat->OperatorRangeMap()));

   x0->Random();
   x1->Random();
   b->PutScalar(0.0);

   // sanity check
   // EpetraExt::RowMatrixToMatrixMarketFile("mat_output.mm",*mat);

   // build Thyra wrappers
   RCP<const Thyra::LinearOpBase<double> >
      tA = Thyra::epetraLinearOp( mat );
   RCP<Thyra::VectorBase<double> >
      tx0 = Thyra::create_Vector( x0, tA->domain() );
   RCP<Thyra::VectorBase<double> >
      tx1 = Thyra::create_Vector( x1, tA->domain() );
   RCP<const Thyra::VectorBase<double> >
      tb = Thyra::create_Vector( b, tA->range() );

   // now comes Stratimikos
   RCP<Teuchos::ParameterList> paramList = Teuchos::getParametersFromXmlFile("BelosGCRODRTest.xml");
   Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
   linearSolverBuilder.setParameterList(paramList);
 
   // Create a linear solver factory given information read from the
   // parameter list.
   RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory =
         linearSolverBuilder.createLinearSolveStrategy("");

   // Create a linear solver based on the forward operator A
   RCP<Thyra::LinearOpWithSolveBase<double> > lows =
         Thyra::linearOpWithSolve(*lowsFactory, tA);

   // Solve the linear system 
   Thyra::SolveStatus<double> status; 
   status = Thyra::solve<double>(*lows, Thyra::NOTRANS, *tb, tx0.ptr());
   status = Thyra::solve<double>(*lows, Thyra::NOTRANS, *tb, tx1.ptr());
}
Beispiel #8
0
  virtual
  Teuchos::RCP<const Thyra::LinearOpWithSolveFactoryBase<double>>
  get_W_factory() const
  {
    Stratimikos::DefaultLinearSolverBuilder builder;
    auto p = Teuchos::rcp(new Teuchos::ParameterList());
    mikado::std_map_to_teuchos_list(
        mikado::convert_to_belos_parameters(this->linear_solver_params_),
        *p
        );
    builder.setParameterList(p);

    auto lowsFactory = builder.createLinearSolveStrategy("");

    lowsFactory->setVerbLevel(Teuchos::VERB_LOW);

    return lowsFactory;
  }
TEUCHOS_UNIT_TEST(tStratimikosFactory, test_MultipleCalls) 
{
  using Teuchos::RCP;
  using Teuchos::ParameterList;

  // build global (or serial communicator)
  #ifdef HAVE_MPI
   Epetra_MpiComm comm(MPI_COMM_WORLD);
  #else
   Epetra_SerialComm comm;
  #endif

  // build epetra operator
  RCP<Epetra_Operator> eA = buildStridedSystem(comm,5);
  RCP<Thyra::LinearOpBase<double> > tA = Thyra::nonconstEpetraLinearOp(eA);

  Stratimikos::DefaultLinearSolverBuilder stratFactory;
  RCP<ParameterList> params = Teuchos::rcp(new ParameterList(*stratFactory.getValidParameters()));
  ParameterList & tekoList = params->sublist("Preconditioner Types").sublist("Teko");
  tekoList.set("Write Block Operator", false);
  tekoList.set("Test Block Operator", false);
  tekoList.set("Strided Blocking","1 1");
  tekoList.set("Inverse Type","BGS");
  ParameterList & ifl = tekoList.sublist("Inverse Factory Library");
  ifl.sublist("BGS").set("Type","Block Gauss-Seidel");
  ifl.sublist("BGS").set("Inverse Type","Amesos");

  Teko::addTekoToStratimikosBuilder(stratFactory,"Teko");
  stratFactory.setParameterList(params);
  RCP<Thyra::PreconditionerFactoryBase<double> > precFactory
        = stratFactory.createPreconditioningStrategy("Teko");

  // build teko preconditioner factory
  RCP<Thyra::PreconditionerBase<double> > prec_a = Thyra::prec<double>(*precFactory,tA);
  Teko::LinearOp precOp_a = prec_a->getUnspecifiedPrecOp();
  TEST_ASSERT(precOp_a!=Teuchos::null);

  // try to do it again
  RCP<Thyra::PreconditionerBase<double> > prec_b = Thyra::prec<double>(*precFactory,tA);
  Teko::LinearOp precOp_b = prec_b->getUnspecifiedPrecOp();
  TEST_ASSERT(precOp_b!=Teuchos::null);
}
int main(int argc, char* argv[])
{

    using Teuchos::describe;
    using Teuchos::rcp;
    using Teuchos::rcp_dynamic_cast;
    using Teuchos::rcp_const_cast;
    using Teuchos::RCP;
    using Teuchos::CommandLineProcessor;
    using Teuchos::ParameterList;
    using Teuchos::sublist;
    using Teuchos::getParametersFromXmlFile;
    typedef ParameterList::PrintOptions PLPrintOptions;
    using Thyra::inverse;
    using Thyra::initializePreconditionedOp;
    using Thyra::initializeOp;
    using Thyra::unspecifiedPrec;
    using Thyra::solve;
    typedef RCP<const Thyra::LinearOpBase<double> > LinearOpPtr;
    typedef RCP<Thyra::VectorBase<double> > VectorPtr;

    bool success = true;
    bool verbose = true;

    Teuchos::GlobalMPISession mpiSession(&argc,&argv);

    Teuchos::RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

    try {

        //
        // Read in options from the command line
        //

        CommandLineProcessor clp(false); // Don't throw exceptions

        const int numVerbLevels = 6;
        Teuchos::EVerbosityLevel
        verbLevelValues[] =
        {
            Teuchos::VERB_DEFAULT, Teuchos::VERB_NONE,
            Teuchos::VERB_LOW, Teuchos::VERB_MEDIUM,
            Teuchos::VERB_HIGH, Teuchos::VERB_EXTREME
        };
        const char*
        verbLevelNames[] =
        { "default", "none", "low", "medium", "high", "extreme" };

        Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_MEDIUM;
        clp.setOption( "verb-level", &verbLevel,
                       numVerbLevels, verbLevelValues, verbLevelNames,
                       "Verbosity level used for all objects."
                     );

        std::string matrixFile = ".";
        clp.setOption( "matrix-file", &matrixFile,
                       "Matrix file."
                     );

        std::string paramListFile = "";
        clp.setOption( "param-list-file", &paramListFile,
                       "Parameter list for preconditioner and solver blocks."
                     );

        bool showParams = false;
        clp.setOption( "show-params", "no-show-params", &showParams,
                       "Show the parameter list or not."
                     );

        bool testPrecIsLinearOp = true;
        clp.setOption( "test-prec-is-linear-op", "test-prec-is-linear-op", &testPrecIsLinearOp,
                       "Test if the preconditioner is a linear operator or not."
                     );

        double solveTol = 1e-8;
        clp.setOption( "solve-tol", &solveTol,
                       "Tolerance for the solution to determine success or failure!"
                     );

        clp.setDocString(
            "This example program shows how to use one linear solver (e.g. AztecOO)\n"
            "as a preconditioner for another iterative solver (e.g. Belos).\n"
        );

        // Note: Use --help on the command line to see the above documentation

        CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
        if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;


        //
        *out << "\nA) Reading in the matrix ...\n";
        //

#ifdef HAVE_MPI
        Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
        Epetra_SerialComm comm;
#endif

        const LinearOpPtr A = readEpetraCrsMatrixFromMatrixMarketAsLinearOp(
                                  matrixFile, comm, "A");
        *out << "\nA = " << describe(*A,verbLevel) << "\n";

        const RCP<ParameterList> paramList = getParametersFromXmlFile(paramListFile);
        if (showParams) {
            *out << "\nRead in parameter list:\n\n";
            paramList->print(*out, PLPrintOptions().indent(2).showTypes(true));
        }

        //
        *out << "\nB) Get the preconditioner as a forward solver\n";
        //

        const RCP<ParameterList> precParamList = sublist(paramList, "Preconditioner Solver");
        Stratimikos::DefaultLinearSolverBuilder precSolverBuilder;
        precSolverBuilder.setParameterList(precParamList);
        const RCP<const Thyra::LinearOpWithSolveFactoryBase<double> > precSolverStrategy
            = createLinearSolveStrategy(precSolverBuilder);
        //precSolverStrategy->setVerbLevel(verbLevel);

        const LinearOpPtr A_inv_prec = inverse<double>(*precSolverStrategy, A,
                                       Thyra::SUPPORT_SOLVE_FORWARD_ONLY,
                                       Teuchos::null, // Use internal solve criteria
                                       Thyra::IGNORE_SOLVE_FAILURE // Ignore solve failures since this is just a prec
                                                      );
        *out << "\nA_inv_prec = " << describe(*A_inv_prec, verbLevel) << "\n";

        if (testPrecIsLinearOp) {
            *out << "\nTest that the preconditioner A_inv_prec is indeed a linear operator.\n";
            Thyra::LinearOpTester<double> linearOpTester;
            linearOpTester.check_adjoint(false);
            const bool linearOpCheck = linearOpTester.check(*A_inv_prec, out.ptr());
            if (!linearOpCheck) {
                success = false;
            }
        }

        //
        *out << "\nC) Create the forward solver using the created preconditioner ...\n";
        //

        const RCP<ParameterList> fwdSolverParamList = sublist(paramList, "Forward Solver");
        Stratimikos::DefaultLinearSolverBuilder fwdSolverSolverBuilder;
        fwdSolverSolverBuilder.setParameterList(fwdSolverParamList);
        const RCP<const Thyra::LinearOpWithSolveFactoryBase<double> > fwdSolverSolverStrategy
            = createLinearSolveStrategy(fwdSolverSolverBuilder);

        const RCP<Thyra::LinearOpWithSolveBase<double> >
        A_lows = fwdSolverSolverStrategy->createOp();

        initializePreconditionedOp<double>( *fwdSolverSolverStrategy, A,
                                            unspecifiedPrec(A_inv_prec), A_lows.ptr());
        //A_lows->setVerbLevel(verbLevel);
        *out << "\nA_lows = " << describe(*A_lows, verbLevel) << "\n";

        //
        *out << "\nD) Solve the linear system for a random RHS ...\n";
        //

        VectorPtr x = createMember(A->domain());
        VectorPtr b = createMember(A->range());
        Thyra::randomize(-1.0, +1.0, b.ptr());
        Thyra::assign(x.ptr(), 0.0); // Must give an initial guess!

        Thyra::SolveStatus<double>
        solveStatus = solve<double>( *A_lows, Thyra::NOTRANS, *b, x.ptr() );

        *out << "\nSolve status:\n" << solveStatus;

        *out << "\nSolution ||x|| = " << Thyra::norm(*x) << "\n";

        if(showParams) {
            *out << "\nParameter list after use:\n\n";
            paramList->print(*out, PLPrintOptions().indent(2).showTypes(true));
        }

        //
        *out << "\nF) Checking the error in the solution of r=b-A*x ...\n";
        //

        VectorPtr Ax = Thyra::createMember(b->space());
        Thyra::apply( *A, Thyra::NOTRANS, *x, Ax.ptr() );
        VectorPtr r = Thyra::createMember(b->space());
        Thyra::V_VmV<double>(r.ptr(), *b, *Ax);

        double
        Ax_nrm = Thyra::norm(*Ax),
        r_nrm = Thyra::norm(*r),
        b_nrm = Thyra::norm(*b),
        r_nrm_over_b_nrm = r_nrm / b_nrm;

        bool resid_tol_check = ( r_nrm_over_b_nrm <= solveTol );
        if(!resid_tol_check) success = false;

        *out
                << "\n||A*x|| = " << Ax_nrm << "\n";

        *out
                << "\n||A*x-b||/||b|| = " << r_nrm << "/" << b_nrm
                << " = " << r_nrm_over_b_nrm << " <= " << solveTol
                << " : " << Thyra::passfail(resid_tol_check) << "\n";

        Teuchos::TimeMonitor::summarize(*out<<"\n");
    }
    TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success)

    if (verbose) {
        if(success)  *out << "\nCongratulations! All of the tests checked out!\n";
        else         *out << "\nOh no! At least one of the tests failed!\n";
    }

    return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
  void solvetriadmatrixwithtrilinos(int& nnz, int& order, int* row, 
              int* col, double* val, double* rhs, double* solution) {
#else
  void solvetriadmatrixwithtrilinos_(int& nnz, int& order, int* row, 
              int* col, double* val, double* rhs, double* solution) {
#endif

    try{
    
#ifdef _MPI
    Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
    Epetra_SerialComm Comm;
#endif
    
    int i, j, ierr;
    int MyPID = Comm.MyPID();
    bool verbose = (MyPID == 0);
    Epetra_Map RowMap(order, 0, Comm);
    int NumMyElements = RowMap.NumMyElements();
    int *MyGlobalElements = new int[NumMyElements];
    RowMap.MyGlobalElements(&MyGlobalElements[0]);

#ifdef _MPI
    int nPEs;
    MPI_Comm_size(MPI_COMM_WORLD, &nPEs);
#endif

    int anEst = nnz / order + 1;
    Epetra_CrsMatrix A(Copy, RowMap, anEst);
    
    for (j=0; j<nnz; ++j) {
      if (RowMap.MyGID(row[j]) ) {
	ierr = A.InsertGlobalValues(row[j], 1, &(val[j]), &(col[j]) );
	assert(ierr >= 0);
      }
    }
    
    ierr = A.FillComplete();
    assert(ierr == 0);
    
    //-------------------------------------------------------------------------
    // RN_20091221: Taking care of the rhs
    //-------------------------------------------------------------------------
    Epetra_Vector b(RowMap);

    // Inserting values into the rhs
    double *MyGlobalValues = new double[NumMyElements];
    for (j=0; j<NumMyElements; ++j) {
      MyGlobalValues[j] = rhs[MyGlobalElements[j] ];
    }
    ierr = b.ReplaceGlobalValues(NumMyElements, &MyGlobalValues[0],
				 &MyGlobalElements[0]);

    //-------------------------------------------------------------------------
    // RN_20091221: Taking care of the solution
    //-------------------------------------------------------------------------
    Epetra_Vector x(RowMap);

    Teuchos::ParameterList paramList;

    Teuchos::RCP<Teuchos::ParameterList>
      paramList1 = Teuchos::rcp(&paramList, false);
    Teuchos::updateParametersFromXmlFile("./strat1.xml", paramList1.get() );

    Teuchos::RCP<Teuchos::FancyOStream>
      out = Teuchos::VerboseObjectBase::getDefaultOStream();

    Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;

    Teuchos::RCP<Epetra_CrsMatrix> epetraOper = Teuchos::rcp(&A, false);
    Teuchos::RCP<Epetra_Vector> epetraRhs = Teuchos::rcp(&b, false);
    Teuchos::RCP<Epetra_Vector> epetraSol = Teuchos::rcp(&x, false);

    Teuchos::RCP<const Thyra::LinearOpBase<double> >
      thyraOper = Thyra::epetraLinearOp(epetraOper);
    Teuchos::RCP<Thyra::VectorBase<double> >
      thyraRhs = Thyra::create_Vector(epetraRhs, thyraOper->range() );
    Teuchos::RCP<Thyra::VectorBase<double> >
      thyraSol = Thyra::create_Vector(epetraSol, thyraOper->domain() );

    linearSolverBuilder.setParameterList(Teuchos::rcp(&paramList, false) );

    Teuchos::RCP<Thyra::LinearOpWithSolveFactoryBase<double> >
      lowsFactory = linearSolverBuilder.createLinearSolveStrategy("");

    lowsFactory->setOStream(out);
    lowsFactory->setVerbLevel(Teuchos::VERB_LOW);

    Teuchos::RCP<Thyra::LinearOpWithSolveBase<double> >
      lows = Thyra::linearOpWithSolve(*lowsFactory, thyraOper);

    Thyra::SolveStatus<double>
      status = Thyra::solve(*lows, Thyra::NOTRANS, *thyraRhs, &*thyraSol);

    thyraSol = Teuchos::null;

    // For debugging =)
    //    cout << "A: " << A << endl;
    //    cout << "b: " << b << endl;
    //    cout << "x: " << x << endl;

    //    Epetra_Vector temp(RowMap);
    //    double residualNorm;
    //    A.Multiply(false, x, temp);
    //    temp.Update(-1, b, 1);
    //    temp.Norm2(&residualNorm);

    Epetra_LocalMap localMap(order, 0, Comm);
    Epetra_Vector xExtra(localMap); // local vector in each processor

    // RN_20091218: Create an import map and then import the data to the
    // local vector.
    Epetra_Import import(localMap, RowMap);

    xExtra.Import(x, import, Add);
    xExtra.ExtractCopy(solution);

    delete[] MyGlobalElements;
    delete[] MyGlobalValues;
    }
  
    catch (std::exception& e) {
      cout << e.what() << endl;
    }
    catch (string& s) {
      cout << s << endl;
    }
    catch (char *s) {
      cout << s << endl;
    }
    catch (...) {
      cout << "Caught unknown exception!" << endl;
    }
  }

} // extern "C"
void Piro::RythmosSolver<Scalar>::initialize(
#endif
    const Teuchos::RCP<Teuchos::ParameterList> &appParams,
    const Teuchos::RCP< Thyra::ModelEvaluator<Scalar> > &in_model,
    const Teuchos::RCP<Rythmos::IntegrationObserverBase<Scalar> > &observer)
{

    using Teuchos::ParameterList;
    using Teuchos::parameterList;
    using Teuchos::RCP;
    using Teuchos::rcp;

    // set some internals
    model = in_model;
    num_p = in_model->Np();
    num_g = in_model->Ng();

    //
    *out << "\nA) Get the base parameter list ...\n";
    //


    if (appParams->isSublist("Rythmos")) {
        RCP<Teuchos::ParameterList> rythmosPL = sublist(appParams, "Rythmos", true);
        rythmosPL->validateParameters(*getValidRythmosParameters(),0);

        {
            const std::string verbosity = rythmosPL->get("Verbosity Level", "VERB_DEFAULT");
            if      (verbosity == "VERB_NONE")    solnVerbLevel = Teuchos::VERB_NONE;
            else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT;
            else if (verbosity == "VERB_LOW")     solnVerbLevel = Teuchos::VERB_LOW;
            else if (verbosity == "VERB_MEDIUM")  solnVerbLevel = Teuchos::VERB_MEDIUM;
            else if (verbosity == "VERB_HIGH")    solnVerbLevel = Teuchos::VERB_HIGH;
            else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME;
            else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Unknown verbosity option specified in Piro_RythmosSolver.");
        }

        t_initial = rythmosPL->get("Initial Time", 0.0);
        t_final = rythmosPL->get("Final Time", 0.1);

        const std::string stepperType = rythmosPL->get("Stepper Type", "Backward Euler");

        //
        *out << "\nC) Create and initalize the forward model ...\n";
        //

        *out << "\nD) Create the stepper and integrator for the forward problem ...\n";
        //

        if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "Rythmos") {
            Teuchos::RCP<Rythmos::TimeStepNonlinearSolver<Scalar> > rythmosTimeStepSolver =
                Rythmos::timeStepNonlinearSolver<Scalar>();
            if (rythmosPL->getEntryPtr("NonLinear Solver")) {
                RCP<Teuchos::ParameterList> nonlinePL =
                    sublist(rythmosPL, "NonLinear Solver", true);
                rythmosTimeStepSolver->setParameterList(nonlinePL);
            }
            fwdTimeStepSolver = rythmosTimeStepSolver;
        }
        else if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "NOX") {
#ifdef HAVE_PIRO_NOX
            Teuchos::RCP<Thyra::NOXNonlinearSolver> nox_solver =  Teuchos::rcp(new Thyra::NOXNonlinearSolver);
            Teuchos::RCP<Teuchos::ParameterList> nox_params = Teuchos::rcp(new Teuchos::ParameterList);
            *nox_params = appParams->sublist("NOX");
            nox_solver->setParameterList(nox_params);
            fwdTimeStepSolver = nox_solver;
#else
            TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Requested NOX solver for a Rythmos Transient solve, Trilinos was not built with NOX enabled.  Please rebuild Trilinos or use the native Rythmos nonlinear solver.");
#endif

        }

        if (stepperType == "Backward Euler") {
            fwdStateStepper = Rythmos::backwardEulerStepper<Scalar> (model, fwdTimeStepSolver);
            fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
        }
        else if (stepperType == "Forward Euler") {
            fwdStateStepper = Rythmos::forwardEulerStepper<Scalar> (model);
            fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
        }
        else if (stepperType == "Explicit RK") {
            fwdStateStepper = Rythmos::explicitRKStepper<Scalar>(model);
            fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
        }
        else if (stepperType == "BDF") {
            Teuchos::RCP<Teuchos::ParameterList> BDFparams =
                Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);
            Teuchos::RCP<Teuchos::ParameterList> BDFStepControlPL =
                Teuchos::sublist(BDFparams,"Step Control Settings");

            fwdStateStepper = Teuchos::rcp( new Rythmos::ImplicitBDFStepper<Scalar>(model,fwdTimeStepSolver,BDFparams) );
            fwdStateStepper->setInitialCondition(model->getNominalValues());

        }
        else {
            // first (before failing) check to see if the user has added stepper factory
            typename std::map<std::string,Teuchos::RCP<Piro::RythmosStepperFactory<Scalar> > >::const_iterator
            stepFactItr = stepperFactories.find(stepperType);
            if(stepFactItr!=stepperFactories.end()) {
                // the user has added it, hot dog lets build a new stepper!
                Teuchos::RCP<Teuchos::ParameterList> stepperParams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);

                // build the stepper using the factory
                fwdStateStepper = stepFactItr->second->buildStepper(model,fwdTimeStepSolver,stepperParams);

                // the user decided to override the model being used (let them)
                if(fwdStateStepper->getModel()!=model && fwdStateStepper->getModel()!=Teuchos::null) {
                    model = Teuchos::rcp_const_cast<Thyra::ModelEvaluator<Scalar> >(fwdStateStepper->getModel());

                    num_p = in_model->Np();
                    num_g = in_model->Ng();
                }
            }
            else {
                TEUCHOS_TEST_FOR_EXCEPTION(
                    true, Teuchos::Exceptions::InvalidParameter,
                    std::endl << "Error! Piro::RythmosSolver: Invalid Steper Type: "
                    << stepperType << std::endl);
            }
        }

        // Step control strategy
        {
            // If the stepper can accept a step control strategy, then attempt to build one.
            RCP<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> > scsa_stepper =
                Teuchos::rcp_dynamic_cast<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> >(fwdStateStepper);

            if (Teuchos::nonnull(scsa_stepper)) {
                const std::string step_control_strategy = rythmosPL->get("Step Control Strategy Type", "None");

                if (step_control_strategy == "None") {
                    // don't do anything, stepper will build default
                } else if (step_control_strategy == "ImplicitBDFRamping") {

                    const RCP<Rythmos::ImplicitBDFStepperRampingStepControl<Scalar> > rscs =
                        rcp(new Rythmos::ImplicitBDFStepperRampingStepControl<Scalar>);

                    const RCP<ParameterList> p = parameterList(rythmosPL->sublist("Rythmos Step Control Strategy"));
                    rscs->setParameterList(p);

                    scsa_stepper->setStepControlStrategy(rscs);
                }
                else {
                    // first (before failing) check to see if the user has added step control factory
                    typename std::map<std::string,Teuchos::RCP<Piro::RythmosStepControlFactory<Scalar> > >::const_iterator
                    stepControlFactItr = stepControlFactories.find(step_control_strategy);
                    if (stepControlFactItr != stepControlFactories.end())
                    {

                        const RCP<Rythmos::StepControlStrategyBase<Scalar> > rscs = stepControlFactItr->second->buildStepControl();

                        const RCP<ParameterList> p = parameterList(rythmosPL -> sublist("Rythmos Step Control Strategy"));

                        rscs->setParameterList(p);

                        scsa_stepper->setStepControlStrategy(rscs);
                    }
                    else {
                        TEUCHOS_TEST_FOR_EXCEPTION(
                            true, std::logic_error,
                            "Error! Piro::RythmosSolver: Invalid step control strategy type: "
                            << step_control_strategy << std::endl);
                    }
                }
            }
        }
        {
            const RCP<Teuchos::ParameterList> integrationControlPL =
                Teuchos::sublist(rythmosPL, "Rythmos Integration Control", true);

            RCP<Rythmos::DefaultIntegrator<Scalar> > defaultIntegrator;
            if (rythmosPL->get("Rythmos Integration Control Strategy", "Simple") == "Simple") {
                defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL));
            }
            else if(rythmosPL->get<std::string>("Rythmos Integration Control Strategy") == "Ramping") {
                defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::rampingIntegrationControlStrategy<Scalar>(integrationControlPL));
            }
            fwdStateIntegrator = defaultIntegrator;
        }

        fwdStateIntegrator->setParameterList(sublist(rythmosPL, "Rythmos Integrator", true));

        if (Teuchos::nonnull(observer)) {
            fwdStateIntegrator->setIntegrationObserver(observer);
        }
    }

    else if (appParams->isSublist("Rythmos Solver")) {
        /** New parameter list format **/
        RCP<Teuchos::ParameterList> rythmosSolverPL = sublist(appParams, "Rythmos Solver", true);
        RCP<Teuchos::ParameterList> rythmosPL = sublist(rythmosSolverPL, "Rythmos", true);

        {
            const std::string verbosity = rythmosSolverPL->get("Verbosity Level", "VERB_DEFAULT");
            if      (verbosity == "VERB_NONE")    solnVerbLevel = Teuchos::VERB_NONE;
            else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT;
            else if (verbosity == "VERB_LOW")     solnVerbLevel = Teuchos::VERB_LOW;
            else if (verbosity == "VERB_MEDIUM")  solnVerbLevel = Teuchos::VERB_MEDIUM;
            else if (verbosity == "VERB_HIGH")    solnVerbLevel = Teuchos::VERB_HIGH;
            else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME;
            else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
                                                "Unknown verbosity option specified in Piro_RythmosSolver.");
        }

        t_initial = rythmosPL->sublist("Integrator Settings").get("Initial Time", 0.0);
        t_final = rythmosPL->sublist("Integrator Settings").get("Final Time", 0.1);

        const std::string stepperType = rythmosPL->sublist("Stepper Settings")
                                        .sublist("Stepper Selection").get("Stepper Type", "Backward Euler");
        //
        //    *out << "\nB) Create the Stratimikos linear solver factory ...\n";
        //
        // This is the linear solve strategy that will be used to solve for the
        // linear system with the W.
        //
        Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;

#ifdef HAVE_PIRO_IFPACK2
        typedef Thyra::PreconditionerFactoryBase<double> Base;
#ifdef ALBANY_BUILD
        typedef Thyra::Ifpack2PreconditionerFactory<Tpetra::CrsMatrix<double, LocalOrdinal, GlobalOrdinal, Node> > Impl;
#else
        typedef Thyra::Ifpack2PreconditionerFactory<Tpetra::CrsMatrix<double> > Impl;
#endif
        linearSolverBuilder.setPreconditioningStrategyFactory(Teuchos::abstractFactoryStd<Base, Impl>(), "Ifpack2");
#endif
#ifdef HAVE_PIRO_MUELU
#ifdef ALBANY_BUILD
        Stratimikos::enableMueLu<LocalOrdinal, GlobalOrdinal, Node>(linearSolverBuilder);
#else
        Stratimikos::enableMueLu(linearSolverBuilder);
#endif
#endif

        linearSolverBuilder.setParameterList(sublist(rythmosSolverPL, "Stratimikos", true));
        rythmosSolverPL->validateParameters(*getValidRythmosSolverParameters(),0);
        RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory =
            createLinearSolveStrategy(linearSolverBuilder);
        //
        *out << "\nC) Create and initalize the forward model ...\n";
        //
        // C.1) Create the underlying EpetraExt::ModelEvaluator
        // already constructed as "model". Decorate if needed.
        // TODO: Generelize to any explicit method, option to invert mass matrix
        if (stepperType == "Explicit RK") {
            if (rythmosSolverPL->get("Invert Mass Matrix", false)) {
                Teuchos::RCP<Thyra::ModelEvaluator<Scalar> > origModel = model;
                rythmosSolverPL->get("Lump Mass Matrix", false);  //JF line does not do anything
                model = Teuchos::rcp(new Piro::InvertMassMatrixDecorator<Scalar>(
                                         sublist(rythmosSolverPL,"Stratimikos", true), origModel,
                                         true,rythmosSolverPL->get("Lump Mass Matrix", false),false));
            }
        }
        // C.2) Create the Thyra-wrapped ModelEvaluator

        thyraModel = rcp(new Thyra::DefaultModelEvaluatorWithSolveFactory<Scalar>(model, lowsFactory));

        const RCP<const Thyra::VectorSpaceBase<double> > x_space =
            thyraModel->get_x_space();

        //
        *out << "\nD) Create the stepper and integrator for the forward problem ...\n";
        //
        fwdTimeStepSolver = Rythmos::timeStepNonlinearSolver<double>();

        if (rythmosSolverPL->getEntryPtr("NonLinear Solver")) {
            const RCP<Teuchos::ParameterList> nonlinePL =
                sublist(rythmosSolverPL, "NonLinear Solver", true);
            fwdTimeStepSolver->setParameterList(nonlinePL);
        }
        // Force Default Integrator since this is needed for Observers
        rythmosPL->sublist("Integrator Settings").sublist("Integrator Selection").
        set("Integrator Type","Default Integrator");

        RCP<Rythmos::IntegratorBuilder<double> > ib = Rythmos::integratorBuilder<double>();
        ib->setParameterList(rythmosPL);
        Thyra::ModelEvaluatorBase::InArgs<double> ic = thyraModel->getNominalValues();
        RCP<Rythmos::IntegratorBase<double> > integrator = ib->create(thyraModel,ic,fwdTimeStepSolver);
        fwdStateIntegrator = Teuchos::rcp_dynamic_cast<Rythmos::DefaultIntegrator<double> >(integrator,true);

        fwdStateStepper = fwdStateIntegrator->getNonconstStepper();

        if (Teuchos::nonnull(observer))
            fwdStateIntegrator->setIntegrationObserver(observer);

    }
    else {
        TEUCHOS_TEST_FOR_EXCEPTION(
            appParams->isSublist("Rythmos") || appParams->isSublist("Rythmos Solver"),
            Teuchos::Exceptions::InvalidParameter, std::endl <<
            "Error! Piro::RythmosSolver: must have either Rythmos or Rythmos Solver sublist ");

    }

    isInitialized = true;
}
int main(int argc, char *argv[])
{

  using std::endl;
  typedef double Scalar;
  typedef double ScalarMag;
  using Teuchos::describe;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::rcp_implicit_cast;
  using Teuchos::rcp_dynamic_cast;
  using Teuchos::as;
  using Teuchos::ParameterList;
  using Teuchos::CommandLineProcessor;
  typedef Teuchos::ParameterList::PrintOptions PLPrintOptions;
  typedef Thyra::ModelEvaluatorBase MEB;
  typedef Thyra::DefaultMultiVectorProductVectorSpace<Scalar> DMVPVS;
  using Thyra::productVectorBase;

  bool result, success = true;

  Teuchos::GlobalMPISession mpiSession(&argc,&argv);

  RCP<Epetra_Comm> epetra_comm;
#ifdef HAVE_MPI
  epetra_comm = rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
  epetra_comm = rcp( new Epetra_SerialComm );
#endif // HAVE_MPI

  RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

  try {

    //
    // Read commandline options
    //

    CommandLineProcessor clp;
    clp.throwExceptions(false);
    clp.addOutputSetupOptions(true);

    std::string paramsFileName = "";
    clp.setOption( "params-file", &paramsFileName,
      "File name for XML parameters" );

    std::string extraParamsString = "";
    clp.setOption( "extra-params", &extraParamsString,
      "Extra XML parameters" );

    std::string extraParamsFile = "";
    clp.setOption( "extra-params-file", &extraParamsFile, "File containing extra parameters in XML format.");

    double maxStateError = 1e-6;
    clp.setOption( "max-state-error", &maxStateError,
      "The maximum allowed error in the integrated state in relation to the exact state solution" );

    double finalTime = 1e-3;
    clp.setOption( "final-time", &finalTime,
      "Final integration time (initial time is 0.0)" );

    int numTimeSteps = 10;
    clp.setOption( "num-time-steps", &numTimeSteps,
      "Number of (fixed) time steps.  If <= 0.0, then variable time steps are taken" );

    bool useBDF = false;
    clp.setOption( "use-BDF", "use-BE", &useBDF,
      "Use BDF or Backward Euler (BE)" );

    bool useIRK = false;
    clp.setOption( "use-IRK", "use-other", &useIRK,
      "Use IRK or something" );

    bool doFwdSensSolve = false;
    clp.setOption( "fwd-sens-solve", "state-solve", &doFwdSensSolve,
      "Do the forward sensitivity solve or just the state solve" );

    bool doFwdSensErrorControl = false;
    clp.setOption( "fwd-sens-err-cntrl", "no-fwd-sens-err-cntrl", &doFwdSensErrorControl,
      "Do error control on the forward sensitivity solve or not" );

    double maxRestateError = 0.0;
    clp.setOption( "max-restate-error", &maxRestateError,
      "The maximum allowed error between the state integrated by itself verses integrated along with DxDp" );

    double maxSensError = 1e-4;
    clp.setOption( "max-sens-error", &maxSensError,
      "The maximum allowed error in the integrated sensitivity in relation to"
      " the finite-difference sensitivity" );

    Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_DEFAULT;
    setVerbosityLevelOption( "verb-level", &verbLevel,
      "Top-level verbosity level.  By default, this gets deincremented as you go deeper into numerical objects.",
      &clp );

    bool testExactSensitivity = false;
    clp.setOption( "test-exact-sens", "no-test-exact-sens", &testExactSensitivity,
      "Test the exact sensitivity with finite differences or not." );

    bool dumpFinalSolutions = false;
    clp.setOption(
      "dump-final-solutions", "no-dump-final-solutions", &dumpFinalSolutions,
      "Determine if the final solutions are dumpped or not." );

    CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
    if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;

    if ( Teuchos::VERB_DEFAULT == verbLevel )
      verbLevel = Teuchos::VERB_LOW;

    const Teuchos::EVerbosityLevel
      solnVerbLevel = ( dumpFinalSolutions ? Teuchos::VERB_EXTREME : verbLevel );

    //
    // Get the base parameter list that all other parameter lists will be read
    // from.
    //

    RCP<ParameterList>
      paramList = Teuchos::parameterList();
    if (paramsFileName.length())
      updateParametersFromXmlFile( paramsFileName, paramList.ptr() );
    if(extraParamsFile.length())
      Teuchos::updateParametersFromXmlFile( "./"+extraParamsFile, paramList.ptr() );
    if (extraParamsString.length())
      updateParametersFromXmlString( extraParamsString, paramList.ptr() );

    if (testExactSensitivity) {
      paramList->sublist(DiagonalTransientModel_name).set("Exact Solution as Response",true);
    }

    paramList->validateParameters(*getValidParameters(),0); // Only validate top level lists!

    //
    // Create the Stratimikos linear solver factory.
    //
    // This is the linear solve strategy that will be used to solve for the
    // linear system with the W.
    //

    Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
    linearSolverBuilder.setParameterList(sublist(paramList,Stratimikos_name));
    RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
      W_factory = createLinearSolveStrategy(linearSolverBuilder);

    //
    // Create the underlying EpetraExt::ModelEvaluator
    //

    RCP<EpetraExt::DiagonalTransientModel>
      epetraStateModel = EpetraExt::diagonalTransientModel(
        epetra_comm,
        sublist(paramList,DiagonalTransientModel_name)
        );

    *out <<"\nepetraStateModel valid options:\n";
    epetraStateModel->getValidParameters()->print(
      *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
      );

    //
    // Create the Thyra-wrapped ModelEvaluator
    //

    RCP<Thyra::ModelEvaluator<double> >
      stateModel = epetraModelEvaluator(epetraStateModel,W_factory);

    *out << "\nParameter names = " << *stateModel->get_p_names(0) << "\n";

    //
    // Create the Rythmos stateStepper
    //

    RCP<Rythmos::TimeStepNonlinearSolver<double> >
      nonlinearSolver = Rythmos::timeStepNonlinearSolver<double>();
    RCP<ParameterList>
      nonlinearSolverPL = sublist(paramList,TimeStepNonlinearSolver_name);
    nonlinearSolverPL->get("Default Tol",1e-3*maxStateError); // Set default if not set
    nonlinearSolver->setParameterList(nonlinearSolverPL);

    RCP<Rythmos::StepperBase<Scalar> > stateStepper;

    if (useBDF) {
      stateStepper = rcp(
        new Rythmos::ImplicitBDFStepper<double>(
          stateModel, nonlinearSolver
          )
        );
    }
    else if (useIRK) {
      // We need a separate LOWSFB object for the IRK stepper
      RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
        irk_W_factory = createLinearSolveStrategy(linearSolverBuilder);
      RCP<Rythmos::RKButcherTableauBase<double> > irkbt = Rythmos::createRKBT<double>("Backward Euler");
      stateStepper = Rythmos::implicitRKStepper<double>(
        stateModel, nonlinearSolver, irk_W_factory, irkbt
        );
    }
    else {
      stateStepper = rcp(
        new Rythmos::BackwardEulerStepper<double>(
          stateModel, nonlinearSolver
          )
        );
    }

    *out <<"\nstateStepper:\n" << describe(*stateStepper,verbLevel);
    *out <<"\nstateStepper valid options:\n";
    stateStepper->getValidParameters()->print(
      *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
      );

    stateStepper->setParameterList(sublist(paramList,RythmosStepper_name));

    //
    // Setup finite difference objects that will be used for tests
    //

    Thyra::DirectionalFiniteDiffCalculator<Scalar> fdCalc;
    fdCalc.setParameterList(sublist(paramList,FdCalc_name));
    fdCalc.setOStream(out);
    fdCalc.setVerbLevel(verbLevel);

    //
    // Use a StepperAsModelEvaluator to integrate the state
    //

    const MEB::InArgs<Scalar>
      state_ic = stateModel->getNominalValues();
    *out << "\nstate_ic:\n" << describe(state_ic,verbLevel);

    RCP<Rythmos::IntegratorBase<Scalar> > integrator;
    {
      RCP<ParameterList>
        integratorPL = sublist(paramList,RythmosIntegrator_name);
      integratorPL->set( "Take Variable Steps", as<bool>(numTimeSteps < 0) );
      integratorPL->set( "Fixed dt", as<double>((finalTime - state_ic.get_t())/numTimeSteps) );
      RCP<Rythmos::IntegratorBase<Scalar> >
        defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(
          Rythmos::simpleIntegrationControlStrategy<Scalar>(integratorPL)
          );
      integrator = defaultIntegrator;
    }

    RCP<Rythmos::StepperAsModelEvaluator<Scalar> >
      stateIntegratorAsModel = Rythmos::stepperAsModelEvaluator(
        stateStepper, integrator, state_ic
        );
    stateIntegratorAsModel->setVerbLevel(verbLevel);

    *out << "\nUse the StepperAsModelEvaluator to integrate state x(p,finalTime) ... \n";

    RCP<Thyra::VectorBase<Scalar> > x_final;

    {

      Teuchos::OSTab tab(out);

      x_final = createMember(stateIntegratorAsModel->get_g_space(0));

      eval_g(
        *stateIntegratorAsModel,
        0, *state_ic.get_p(0),
        finalTime,
        0, &*x_final
        );

      *out
        << "\nx_final = x(p,finalTime) evaluated using stateIntegratorAsModel:\n"
        << describe(*x_final,solnVerbLevel);

    }

    //
    // Test the integrated state against the exact analytical state solution
    //

    RCP<const Thyra::VectorBase<Scalar> >
      exact_x_final = create_Vector(
        epetraStateModel->getExactSolution(finalTime),
        stateModel->get_x_space()
        );

    result = Thyra::testRelNormDiffErr(
      "exact_x_final", *exact_x_final, "x_final", *x_final,
      "maxStateError", maxStateError, "warningTol", 1.0, // Don't warn
      &*out, solnVerbLevel
      );
    if (!result) success = false;

    //
    // Solve and test the forward sensitivity computation
    //

    if (doFwdSensSolve) {

      //
      // Create the forward sensitivity stepper
      //

      RCP<Rythmos::ForwardSensitivityStepper<Scalar> > stateAndSensStepper =
        Rythmos::forwardSensitivityStepper<Scalar>();
      if (doFwdSensErrorControl) {
        stateAndSensStepper->initializeDecoupledSteppers(
          stateModel, 0, stateModel->getNominalValues(),
          stateStepper, nonlinearSolver,
          integrator->cloneIntegrator(), finalTime
          );
      }
      else {
        stateAndSensStepper->initializeSyncedSteppers(
          stateModel, 0, stateModel->getNominalValues(),
          stateStepper, nonlinearSolver
          );
        // The above call will result in stateStepper and nonlinearSolver being
        // cloned.  This helps to ensure consistency between the state and
        // sensitivity computations!
      }

      //
      // Set the initial condition for the state and forward sensitivities
      //

      RCP<Thyra::VectorBase<Scalar> > s_bar_init
        = createMember(stateAndSensStepper->getFwdSensModel()->get_x_space());
      assign( s_bar_init.ptr(), 0.0 );
      RCP<Thyra::VectorBase<Scalar> > s_bar_dot_init
        = createMember(stateAndSensStepper->getFwdSensModel()->get_x_space());
      assign( s_bar_dot_init.ptr(), 0.0 );
      // Above, I believe that these are the correct initial conditions for
      // s_bar and s_bar_dot given how the EpetraExt::DiagonalTransientModel
      // is currently implemented!

      RCP<const Rythmos::StateAndForwardSensitivityModelEvaluator<Scalar> >
        stateAndSensModel = stateAndSensStepper->getStateAndFwdSensModel();

      MEB::InArgs<Scalar>
        state_and_sens_ic = stateAndSensStepper->getModel()->createInArgs();

      // Copy time, parameters etc.
      state_and_sens_ic.setArgs(state_ic);
      // Set initial condition for x_bar = [ x; s_bar ]
      state_and_sens_ic.set_x(
        stateAndSensModel->create_x_bar_vec(state_ic.get_x(),s_bar_init)
        );
      // Set initial condition for x_bar_dot = [ x_dot; s_bar_dot ]
      state_and_sens_ic.set_x_dot(
        stateAndSensModel->create_x_bar_vec(state_ic.get_x_dot(),s_bar_dot_init)
        );

      *out << "\nstate_and_sens_ic:\n" << describe(state_and_sens_ic,verbLevel);

      stateAndSensStepper->setInitialCondition(state_and_sens_ic);

      //
      // Use a StepperAsModelEvaluator to integrate the state+sens
      //

      RCP<Rythmos::StepperAsModelEvaluator<Scalar> >
        stateAndSensIntegratorAsModel = Rythmos::stepperAsModelEvaluator(
          rcp_implicit_cast<Rythmos::StepperBase<Scalar> >(stateAndSensStepper),
          integrator, state_and_sens_ic
          );
      stateAndSensIntegratorAsModel->setVerbLevel(verbLevel);

      *out << "\nUse the StepperAsModelEvaluator to integrate state + sens x_bar(p,finalTime) ... \n";

      RCP<Thyra::VectorBase<Scalar> > x_bar_final;

      {

        Teuchos::OSTab tab(out);

        x_bar_final = createMember(stateAndSensIntegratorAsModel->get_g_space(0));

        eval_g(
          *stateAndSensIntegratorAsModel,
          0, *state_ic.get_p(0),
          finalTime,
          0, &*x_bar_final
          );

        *out
          << "\nx_bar_final = x_bar(p,finalTime) evaluated using stateAndSensIntegratorAsModel:\n"
          << describe(*x_bar_final,solnVerbLevel);

      }

      //
      // Test that the state computed above is same as computed initially!
      //

      *out << "\nChecking that x(p,finalTime) computed as part of x_bar above is the same ...\n";

      {

        Teuchos::OSTab tab(out);

        RCP<const Thyra::VectorBase<Scalar> >
          x_in_x_bar_final = productVectorBase<Scalar>(x_bar_final)->getVectorBlock(0);

        result = Thyra::testRelNormDiffErr<Scalar>(
          "x_final", *x_final,
          "x_in_x_bar_final", *x_in_x_bar_final,
          "maxRestateError", maxRestateError,
          "warningTol", 1.0, // Don't warn
          &*out, solnVerbLevel
          );
        if (!result) success = false;

      }

      //
      // Compute DxDp using finite differences
      //

      *out << "\nApproximating DxDp(p,t) using directional finite differences of integrator for x(p,t) ...\n";

      RCP<Thyra::MultiVectorBase<Scalar> > DxDp_fd_final;

      {

        Teuchos::OSTab tab(out);


        MEB::InArgs<Scalar>
          fdBasePoint = stateIntegratorAsModel->createInArgs();

        fdBasePoint.set_t(finalTime);
        fdBasePoint.set_p(0,stateModel->getNominalValues().get_p(0));

        DxDp_fd_final = createMembers(
          stateIntegratorAsModel->get_g_space(0),
          stateIntegratorAsModel->get_p_space(0)->dim()
          );

        typedef Thyra::DirectionalFiniteDiffCalculatorTypes::SelectedDerivatives
          SelectedDerivatives;

        MEB::OutArgs<Scalar> fdOutArgs =
          fdCalc.createOutArgs(
            *stateIntegratorAsModel,
            SelectedDerivatives().supports(MEB::OUT_ARG_DgDp,0,0)
            );
        fdOutArgs.set_DgDp(0,0,DxDp_fd_final);

        // Silence the model evaluators that are called.  The fdCal object
        // will show all of the inputs and outputs for each call.
        stateStepper->setVerbLevel(Teuchos::VERB_NONE);
        stateIntegratorAsModel->setVerbLevel(Teuchos::VERB_NONE);

        fdCalc.calcDerivatives(
          *stateIntegratorAsModel, fdBasePoint,
          stateIntegratorAsModel->createOutArgs(), // Don't bother with function value
          fdOutArgs
          );

        *out
          << "\nFinite difference DxDp_fd_final = DxDp(p,finalTime): "
          << describe(*DxDp_fd_final,solnVerbLevel);

      }

      //
      // Test that the integrated sens and the F.D. sens are similar
      //

      *out << "\nChecking that integrated DxDp(p,finalTime) and finite-diff DxDp(p,finalTime) are similar ...\n";

      {

        Teuchos::OSTab tab(out);

        RCP<const Thyra::VectorBase<Scalar> >
          DxDp_vec_final = Thyra::productVectorBase<Scalar>(x_bar_final)->getVectorBlock(1);

        RCP<const Thyra::VectorBase<Scalar> >
          DxDp_fd_vec_final = Thyra::multiVectorProductVector(
            rcp_dynamic_cast<const Thyra::DefaultMultiVectorProductVectorSpace<Scalar> >(
              DxDp_vec_final->range()
              ),
            DxDp_fd_final
            );

        result = Thyra::testRelNormDiffErr(
          "DxDp_vec_final", *DxDp_vec_final,
          "DxDp_fd_vec_final", *DxDp_fd_vec_final,
          "maxSensError", maxSensError,
          "warningTol", 1.0, // Don't warn
          &*out, solnVerbLevel
          );
        if (!result) success = false;

      }

    }

  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(true,*out,success);

  if(success)
    *out << "\nEnd Result: TEST PASSED" << endl;
  else
    *out << "\nEnd Result: TEST FAILED" << endl;

  return ( success ? 0 : 1 );

} // end main() [Doxygen looks for this!]
TEUCHOS_UNIT_TEST(tStratimikosFactory, test_BlockGaussSeidel) 
{
  using Teuchos::RCP;
  using Teuchos::ParameterList;

  // build global (or serial communicator)
  #ifdef HAVE_MPI
     Epetra_MpiComm comm(MPI_COMM_WORLD);
  #else
     Epetra_SerialComm comm;
  #endif

  // build epetra operator
  RCP<Epetra_Operator> eA = buildStridedSystem(comm,5);
  RCP<Thyra::LinearOpBase<double> > tA = Thyra::nonconstEpetraLinearOp(eA);

  // build stratimikos factory, adding Teko's version
  Stratimikos::DefaultLinearSolverBuilder stratFactory;
  stratFactory.setPreconditioningStrategyFactory(
        Teuchos::abstractFactoryStd<Thyra::PreconditionerFactoryBase<double>,Teko::StratimikosFactory>(),
        "Teko");
  RCP<ParameterList> params = Teuchos::rcp(new ParameterList(*stratFactory.getValidParameters()));
  ParameterList & tekoList = params->sublist("Preconditioner Types").sublist("Teko");
  tekoList.set("Write Block Operator", false);
  tekoList.set("Test Block Operator", false);
  tekoList.set("Strided Blocking","1 1");
  tekoList.set("Inverse Type","BGS");
  ParameterList & ifl = tekoList.sublist("Inverse Factory Library");
  ifl.sublist("BGS").set("Type","Block Gauss-Seidel");
  ifl.sublist("BGS").set("Inverse Type","Amesos");

  // RCP<Thyra::PreconditionerFactoryBase<double> > precFactory
  //       = stratFactory.createPreconditioningStrategy("Teko");

  // build operator to test against
  Teko::LinearOp testOp;
  {
     Teuchos::ParameterList iflCopy(ifl);
     RCP<Epetra_Operator> strided_eA = Teuchos::rcp(new Teko::Epetra::StridedEpetraOperator(2,eA));
     RCP<Teko::InverseLibrary> invLib = Teko::InverseLibrary::buildFromParameterList(iflCopy);
     RCP<const Teko::InverseFactory> invFact = invLib->getInverseFactory("BGS");
     RCP<Teko::Epetra::InverseFactoryOperator> invFactOp = Teuchos::rcp(new Teko::Epetra::InverseFactoryOperator(invFact));
     invFactOp->buildInverseOperator(strided_eA);

     testOp = Thyra::epetraLinearOp(invFactOp,Thyra::NOTRANS,Thyra::EPETRA_OP_APPLY_APPLY_INVERSE);
  }

  stratFactory.setParameterList(params);
  RCP<Thyra::PreconditionerFactoryBase<double> > precFactory
        = stratFactory.createPreconditioningStrategy("Teko");

  // build teko preconditioner factory
  RCP<Thyra::PreconditionerBase<double> > prec = Thyra::prec<double>(*precFactory,tA);
  Teko::LinearOp precOp = prec->getUnspecifiedPrecOp();
  TEST_ASSERT(precOp!=Teuchos::null);

  Thyra::LinearOpTester<double> tester;
  tester.show_all_tests(true);
  tester.set_all_error_tol(0);
  TEST_ASSERT(tester.compare(*precOp,*testOp,Teuchos::ptrFromRef(out)));
}
TEUCHOS_UNIT_TEST(tStratimikosFactory, test_multi_use) 
{
  using Teuchos::RCP;
  using Teuchos::ParameterList;

  // build global (or serial communicator)
  #ifdef HAVE_MPI
     Epetra_MpiComm comm(MPI_COMM_WORLD);
  #else
     Epetra_SerialComm comm;
  #endif

  // build epetra operator
  RCP<Epetra_Operator> eA = buildSystem(comm,5);
  RCP<Thyra::LinearOpBase<double> > tA = Thyra::nonconstEpetraLinearOp(eA);

  // build stratimikos factory, adding Teko's version
  Stratimikos::DefaultLinearSolverBuilder stratFactory;
  stratFactory.setPreconditioningStrategyFactory(
        Teuchos::abstractFactoryStd<Thyra::PreconditionerFactoryBase<double>,Teko::StratimikosFactory>(),
        "Teko");
  RCP<const ParameterList> validParams = stratFactory.getValidParameters();
  stratFactory.setParameterList(Teuchos::rcp(new Teuchos::ParameterList(*validParams)));

  // print out Teko's parameter list and fail if it doesn't exist!
  TEST_NOTHROW(validParams->sublist("Preconditioner Types").sublist("Teko").print(out,
        ParameterList::PrintOptions().showDoc(true).indent(2).showTypes(true)));

  // build teko preconditioner factory
  RCP<Thyra::PreconditionerFactoryBase<double> > precFactory
        = stratFactory.createPreconditioningStrategy("Teko");

  // make sure factory is built
  TEST_ASSERT(precFactory!=Teuchos::null);

  // try using a different preconditioner each time
  RCP<Thyra::PreconditionerBase<double> > prec;
  for(int i=0;i<2;i++) {
     prec = precFactory->createPrec();
     
     RCP<const Thyra::LinearOpSourceBase<double> > losb 
        = rcp(new Thyra::DefaultLinearOpSource<double>(tA));

     precFactory->initializePrec(losb,prec.get());

     RCP<Teko::StratimikosFactory> stratFact =
             rcp_dynamic_cast<Teko::StratimikosFactory>(precFactory);
     const std::vector<int> & decomp = stratFact->getDecomposition();

     TEST_EQUALITY(decomp.size(),1);
     TEST_EQUALITY(decomp[0],1);
  }

  // try using a single preconditioner multiple times
  prec = precFactory->createPrec();
  for(int i=0;i<2;i++) {
     RCP<const Thyra::LinearOpSourceBase<double> > losb 
        = rcp(new Thyra::DefaultLinearOpSource<double>(tA));

     precFactory->initializePrec(losb,prec.get());

     RCP<Teko::StratimikosFactory> stratFact =
             rcp_dynamic_cast<Teko::StratimikosFactory>(precFactory);
     const std::vector<int> & decomp = stratFact->getDecomposition();

     TEST_EQUALITY(decomp.size(),1);
     TEST_EQUALITY(decomp[0],1);
  }

}
int main(int argc, char *argv[])
{

    using std::endl;
    typedef double Scalar;
    typedef double ScalarMag;
    using Teuchos::describe;
    using Teuchos::RCP;
    using Teuchos::rcp;
    using Teuchos::rcp_implicit_cast;
    using Teuchos::rcp_dynamic_cast;
    using Teuchos::as;
    using Teuchos::ParameterList;
    using Teuchos::CommandLineProcessor;
    typedef Teuchos::ParameterList::PrintOptions PLPrintOptions;
    typedef Thyra::ModelEvaluatorBase MEB;
    using Thyra::createMember;
    using Thyra::createMembers;

    bool success = true;

    Teuchos::GlobalMPISession mpiSession(&argc,&argv);

    RCP<Epetra_Comm> epetra_comm;
#ifdef HAVE_MPI
    epetra_comm = rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
    epetra_comm = rcp( new Epetra_SerialComm );
#endif // HAVE_MPI

    RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

    try {

        //
        // A) Read commandline options
        //

        CommandLineProcessor clp;
        clp.throwExceptions(false);
        clp.addOutputSetupOptions(true);

        std::string paramsFileName = "";
        clp.setOption( "params-file", &paramsFileName,
                       "File name for XML parameters" );

        std::string extraParamsString = "";
        clp.setOption( "extra-params", &extraParamsString,
                       "Extra XML parameter string" );

        Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_DEFAULT;
        setVerbosityLevelOption( "verb-level", &verbLevel,
                                 "Top-level verbosity level.  By default, this gets deincremented as you go deeper into numerical objects.",
                                 &clp );

        double finalTime = 1.0;
        clp.setOption( "final-time", &finalTime, "Final time (the inital time)" );

        int numTimeSteps = 2;
        clp.setOption( "num-time-steps", &numTimeSteps, "Number of time steps" );

        bool dumpFinalSolutions = false;
        clp.setOption(
            "dump-final-solutions", "no-dump-final-solutions", &dumpFinalSolutions,
            "Determine if the final solutions are dumpped or not." );

        double maxStateError = 1e-6;
        clp.setOption( "max-state-error", &maxStateError,
                       "The maximum allowed error in the integrated state in relation to the exact state solution" );

        // ToDo: Read in more parameters

        CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
        if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;

        if ( Teuchos::VERB_DEFAULT == verbLevel )
            verbLevel = Teuchos::VERB_LOW;

        const Teuchos::EVerbosityLevel
        solnVerbLevel = ( dumpFinalSolutions ? Teuchos::VERB_EXTREME : verbLevel );

        //
        // B) Get the base parameter list that all other parameter lists will be
        // read from.
        //

        RCP<ParameterList> paramList = Teuchos::parameterList();
        if (paramsFileName.length())
            updateParametersFromXmlFile( paramsFileName, &*paramList );
        if (extraParamsString.length())
            updateParametersFromXmlString( extraParamsString, &*paramList );

        paramList->validateParameters(*getValidParameters());

        //
        // C) Create the Stratimikos linear solver factories.
        //

        // Get the linear solve strategy that will be used to solve for the linear
        // system with the dae's W matrix.
        Stratimikos::DefaultLinearSolverBuilder daeLinearSolverBuilder;
        daeLinearSolverBuilder.setParameterList(sublist(paramList,DAELinearSolver_name));
        RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
        daeLOWSF = createLinearSolveStrategy(daeLinearSolverBuilder);

        // Get the linear solve strategy that can be used to override the overall
        // linear system solve
        Stratimikos::DefaultLinearSolverBuilder overallLinearSolverBuilder;
        overallLinearSolverBuilder.setParameterList(sublist(paramList,OverallLinearSolver_name));
        RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
        overallLOWSF = createLinearSolveStrategy(overallLinearSolverBuilder);

        //
        // D) Create the underlying EpetraExt::ModelEvaluator
        //

        RCP<EpetraExt::DiagonalTransientModel> epetraDaeModel =
            EpetraExt::diagonalTransientModel(
                epetra_comm,
                sublist(paramList,DiagonalTransientModel_name)
            );

        *out <<"\nepetraDaeModel valid options:\n";
        epetraDaeModel->getValidParameters()->print(
            *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
        );

        //
        // E) Create the Thyra-wrapped ModelEvaluator
        //

        RCP<Thyra::ModelEvaluator<double> > daeModel =
            epetraModelEvaluator(epetraDaeModel,daeLOWSF);

        //
        // F) Create the TimeDiscretizedBackwardEulerModelEvaluator
        //

        MEB::InArgs<Scalar> initCond = daeModel->createInArgs();
        initCond.setArgs(daeModel->getNominalValues());

        RCP<Thyra::ModelEvaluator<Scalar> >
        discretizedModel = Rythmos::timeDiscretizedBackwardEulerModelEvaluator<Scalar>(
                               daeModel, initCond, finalTime, numTimeSteps, overallLOWSF );

        *out << "\ndiscretizedModel = " << describe(*discretizedModel,verbLevel);

        //
        // F) Setup a nonlinear solver and solve the system
        //

        // F.1) Setup a nonlinear solver

        Thyra::DampenedNewtonNonlinearSolver<Scalar> nonlinearSolver;
        nonlinearSolver.setOStream(out);
        nonlinearSolver.setVerbLevel(verbLevel);
        //nonlinearSolver.setParameterList(sublist(paramList,NonlinearSolver_name));
        //2007/11/27: rabartl: ToDo: Implement parameter list handling for
        //DampenedNonlinearSolve so that I can uncomment the above line.
        nonlinearSolver.setModel(discretizedModel);

        // F.2) Solve the system

        RCP<Thyra::VectorBase<Scalar> >
        x_bar = createMember(discretizedModel->get_x_space());
        V_S( x_bar.ptr(), 0.0 );

        Thyra::SolveStatus<Scalar> solveStatus =
            Thyra::solve( nonlinearSolver, &*x_bar );

        *out << "\nsolveStatus:\n" << solveStatus;

        *out << "\nx_bar = " << describe(*x_bar,solnVerbLevel);

        //
        // G) Verify that the solution is correct???
        //

        // Check against the end time exact solution.

        RCP<const Thyra::VectorBase<Scalar> >
        exact_x_final = Thyra::create_Vector(
                            epetraDaeModel->getExactSolution(finalTime),
                            daeModel->get_x_space()
                        );

        RCP<const Thyra::VectorBase<Scalar> > solved_x_final
            = rcp_dynamic_cast<Thyra::ProductVectorBase<Scalar> >(x_bar,true)->getVectorBlock(numTimeSteps-1);

        const bool result = Thyra::testRelNormDiffErr(
                                "exact_x_final", *exact_x_final, "solved_x_final", *solved_x_final,
                                "maxStateError", maxStateError, "warningTol", 1.0, // Don't warn
                                &*out, solnVerbLevel
                            );
        if (!result) success = false;

    }
    TEUCHOS_STANDARD_CATCH_STATEMENTS(true,*out,success);

    if(success)
        *out << "\nEnd Result: TEST PASSED" << endl;
    else
        *out << "\nEnd Result: TEST FAILED" << endl;

    return ( success ? 0 : 1 );

} // end main() [Doxygen looks for this!]
// calls MPI_Init and MPI_Finalize
int main(int argc,char * argv[])
{
   using Teuchos::RCP;
   using panzer::StrPureBasisPair;
   using panzer::StrPureBasisComp;

   PHX::InitializeKokkosDevice();

   Teuchos::GlobalMPISession mpiSession(&argc,&argv);
   Teuchos::FancyOStream out(Teuchos::rcpFromRef(std::cout));
   out.setOutputToRootOnly(0);
   out.setShowProcRank(true);

   // variable declarations
   ////////////////////////////////////////////////////

   // factory definitions
   Teuchos::RCP<user_app::MyFactory> eqset_factory = Teuchos::rcp(new user_app::MyFactory);
   panzer_stk::SquareQuadMeshFactory mesh_factory;
   user_app::BCFactory bc_factory;

   // other declarations
   const std::size_t workset_size = 20;
   Teuchos::RCP<panzer::FieldManagerBuilder> fmb = 
         Teuchos::rcp(new panzer::FieldManagerBuilder);

   RCP<panzer_stk::STK_Interface> mesh;

   // construction of uncommitted (no elements) mesh 
   ////////////////////////////////////////////////////////

   // set mesh factory parameters
   RCP<Teuchos::ParameterList> pl = rcp(new Teuchos::ParameterList);
   pl->set("X Blocks",2);
   pl->set("Y Blocks",1);
   pl->set("X Elements",10);
   pl->set("Y Elements",10);
   mesh_factory.setParameterList(pl);
   mesh = mesh_factory.buildUncommitedMesh(MPI_COMM_WORLD);

   // construct input physics and physics block
   ////////////////////////////////////////////////////////

   out << "BUILD PHYSICS" << std::endl;
   Teuchos::RCP<Teuchos::ParameterList> ipb = Teuchos::parameterList("Physics Blocks");
   std::vector<panzer::BC> bcs;
   std::vector<Teuchos::RCP<panzer::PhysicsBlock> > physicsBlocks;
   {
      std::map<std::string,std::string> block_ids_to_physics_ids;
      std::map<std::string,Teuchos::RCP<const shards::CellTopology> > block_ids_to_cell_topo;

      testInitialzation(ipb, bcs);

      block_ids_to_physics_ids["eblock-0_0"] = "test physics";
      block_ids_to_physics_ids["eblock-1_0"] = "test physics";

      block_ids_to_cell_topo["eblock-0_0"] = mesh->getCellTopology("eblock-0_0");
      block_ids_to_cell_topo["eblock-1_0"] = mesh->getCellTopology("eblock-1_0");
   
      Teuchos::RCP<panzer::GlobalData> gd = panzer::createGlobalData();

      int default_integration_order = 1;
      
      // build physicsBlocks map
      panzer::buildPhysicsBlocks(block_ids_to_physics_ids,
                                 block_ids_to_cell_topo,
				 ipb,
				 default_integration_order,
				 workset_size,
			         eqset_factory,
				 gd,
			         true,
			         physicsBlocks);
   }

   // finish building mesh, set required field variables and mesh bulk data
   ////////////////////////////////////////////////////////////////////////

   {
      std::vector<Teuchos::RCP<panzer::PhysicsBlock> >::const_iterator physIter;
      for(physIter=physicsBlocks.begin();physIter!=physicsBlocks.end();++physIter) {
         Teuchos::RCP<const panzer::PhysicsBlock> pb = *physIter;
         const std::vector<StrPureBasisPair> & blockFields = pb->getProvidedDOFs();

         // insert all fields into a set
         std::set<StrPureBasisPair,StrPureBasisComp> fieldNames;
         fieldNames.insert(blockFields.begin(),blockFields.end());

         // add basis to DOF manager: block specific
         std::set<StrPureBasisPair,StrPureBasisComp>::const_iterator fieldItr;
         for (fieldItr=fieldNames.begin();fieldItr!=fieldNames.end();++fieldItr) {

            mesh->addSolutionField(fieldItr->first,pb->elementBlockID());
         }
      }

      mesh_factory.completeMeshConstruction(*mesh,MPI_COMM_WORLD);
   }


   // build worksets
   ////////////////////////////////////////////////////////

   // build worksets
   out << "BUILD WORKSETS" << std::endl;

   Teuchos::RCP<panzer_stk::WorksetFactory> wkstFactory
      = Teuchos::rcp(new panzer_stk::WorksetFactory(mesh)); // build STK workset factory
   Teuchos::RCP<panzer::WorksetContainer> wkstContainer     // attach it to a workset container (uses lazy evaluation)
      = Teuchos::rcp(new panzer::WorksetContainer(wkstFactory,physicsBlocks,workset_size));

   std::vector<std::string> elementBlockNames;
   mesh->getElementBlockNames(elementBlockNames);
   std::map<std::string,Teuchos::RCP<std::vector<panzer::Workset> > > volume_worksets;
   panzer::getVolumeWorksetsFromContainer(*wkstContainer,elementBlockNames,volume_worksets);

   out << "block count = " << volume_worksets.size() << std::endl;
   out << "workset count = " << volume_worksets["eblock-0_0"]->size() << std::endl;
   
   // build DOF Manager
   /////////////////////////////////////////////////////////////
 
   out << "BUILD CONN MANAGER" << std::endl;
   // build the connection manager 
   const Teuchos::RCP<panzer::ConnManager<int,int> > 
     conn_manager = Teuchos::rcp(new panzer_stk::STKConnManager<int>(mesh));

   panzer::DOFManagerFactory<int,int> globalIndexerFactory;
   RCP<panzer::UniqueGlobalIndexer<int,int> > dofManager 
         = globalIndexerFactory.buildUniqueGlobalIndexer(Teuchos::opaqueWrapper(MPI_COMM_WORLD),physicsBlocks,conn_manager);

   // construct some linear algebra object, build object to pass to evaluators
   Teuchos::RCP<const Teuchos::MpiComm<int> > tComm = Teuchos::rcp(new Teuchos::MpiComm<int>(MPI_COMM_WORLD));
   Teuchos::RCP<panzer::EpetraLinearObjFactory<panzer::Traits,int> > eLinObjFactory
         = Teuchos::rcp(new panzer::EpetraLinearObjFactory<panzer::Traits,int>(tComm.getConst(),dofManager));
   Teuchos::RCP<panzer::LinearObjFactory<panzer::Traits> > linObjFactory = eLinObjFactory;

   // setup field manager build
   /////////////////////////////////////////////////////////////
   out << "SETUP FMB" << std::endl;
 
    // Add in the application specific closure model factory
    panzer::ClosureModelFactory_TemplateManager<panzer::Traits> cm_factory; 
    user_app::MyModelFactory_TemplateBuilder cm_builder;
    cm_factory.buildObjects(cm_builder);

    Teuchos::ParameterList closure_models("Closure Models");
    closure_models.sublist("solid").sublist("SOURCE_TEMPERATURE").set<double>("Value",1.0);
    closure_models.sublist("solid").sublist("DENSITY").set<double>("Value",1.0);
    closure_models.sublist("solid").sublist("HEAT_CAPACITY").set<double>("Value",1.0);
    closure_models.sublist("ion solid").sublist("SOURCE_ION_TEMPERATURE").set<double>("Value",1.0);
    closure_models.sublist("ion solid").sublist("ION_DENSITY").set<double>("Value",1.0);
    closure_models.sublist("ion solid").sublist("ION_HEAT_CAPACITY").set<double>("Value",1.0);

    Teuchos::ParameterList user_data("User Data");

    fmb->setWorksetContainer(wkstContainer);
    fmb->setupVolumeFieldManagers(physicsBlocks,cm_factory,closure_models,*linObjFactory,user_data);
    fmb->setupBCFieldManagers(bcs,physicsBlocks,*eqset_factory,cm_factory,bc_factory,closure_models,*linObjFactory,user_data);

   // setup assembly engine
   /////////////////////////////////////////////////////////////

   // build assembly engine
   panzer::AssemblyEngine_TemplateManager<panzer::Traits> ae_tm;
   panzer::AssemblyEngine_TemplateBuilder builder(fmb,linObjFactory);
   ae_tm.buildObjects(builder);

   // setup linear algebra and solve 
   /////////////////////////////////////////////////////////////

   // build ghosted variables
   out << "BUILD LA" << std::endl;
   RCP<panzer::EpetraLinearObjContainer> ghostCont 
         = Teuchos::rcp_dynamic_cast<panzer::EpetraLinearObjContainer>(linObjFactory->buildGhostedLinearObjContainer());
   RCP<panzer::EpetraLinearObjContainer> container 
         = Teuchos::rcp_dynamic_cast<panzer::EpetraLinearObjContainer>(linObjFactory->buildLinearObjContainer());
   eLinObjFactory->initializeContainer(panzer::EpetraLinearObjContainer::X |
                                       panzer::EpetraLinearObjContainer::DxDt |
                                       panzer::EpetraLinearObjContainer::F |
                                       panzer::EpetraLinearObjContainer::Mat,*container);
   eLinObjFactory->initializeGhostedContainer(panzer::EpetraLinearObjContainer::X |
                                              panzer::EpetraLinearObjContainer::DxDt |
                                              panzer::EpetraLinearObjContainer::F |
                                              panzer::EpetraLinearObjContainer::Mat,*ghostCont);

   panzer::AssemblyEngineInArgs input(ghostCont,container);
   input.alpha = 0;
   input.beta = 1;

   // evaluate physics
   out << "EVALUTE" << std::endl;
   ae_tm.getAsObject<panzer::Traits::Residual>()->evaluate(input);
   ae_tm.getAsObject<panzer::Traits::Jacobian>()->evaluate(input);

   out << "RAN SUCCESSFULLY!" << std::endl;

   out << "SOLVE" << std::endl;

   // notice that this should be called by the assembly driver!
   // linObjFactory->ghostToGlobalContainer(*ghostCont,*container);

   Teuchos::RCP<const Thyra::LinearOpBase<double> > th_A = Thyra::epetraLinearOp(container->get_A());
   Teuchos::RCP<const Thyra::VectorSpaceBase<double> > range  = th_A->range();
   Teuchos::RCP<const Thyra::VectorSpaceBase<double> > domain = th_A->domain();

   Teuchos::RCP<Thyra::VectorBase<double> > th_x = Thyra::create_Vector(container->get_x(),domain);
   Teuchos::RCP<Thyra::VectorBase<double> > th_f = Thyra::create_Vector(container->get_f(),range);

   // solve with amesos
   Stratimikos::DefaultLinearSolverBuilder solverBuilder;
   Teuchos::RCP<Teuchos::ParameterList> validList = Teuchos::rcp(new Teuchos::ParameterList(*solverBuilder.getValidParameters()));
   solverBuilder.setParameterList(validList);
   
   RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory = solverBuilder.createLinearSolveStrategy("Amesos");
   RCP<Thyra::LinearOpWithSolveBase<double> > lows = Thyra::linearOpWithSolve(*lowsFactory, th_A.getConst());
   Thyra::solve<double>(*lows, Thyra::NOTRANS, *th_f, th_x.ptr());

   if(false) {
      EpetraExt::RowMatrixToMatrixMarketFile("a_op.mm",*container->get_A());
      EpetraExt::VectorToMatrixMarketFile("x_vec.mm",*container->get_x());
      EpetraExt::VectorToMatrixMarketFile("b_vec.mm",*container->get_f());
   }

   out << "WRITE" << std::endl;

   // redistribute solution vector
   linObjFactory->globalToGhostContainer(*container,*ghostCont,panzer::EpetraLinearObjContainer::X | panzer::EpetraLinearObjContainer::DxDt); 

   panzer_stk::write_solution_data(*dofManager,*mesh,*ghostCont->get_x());
   mesh->writeToExodus("output.exo");

   PHX::FinalizeKokkosDevice();

   return 0;
}
  TEUCHOS_UNIT_TEST(explicit_model_evaluator, basic)
  {
    using Teuchos::RCP;

    PHX::KokkosDeviceSession session;

    bool parameter_on = true;
    Teuchos::RCP<panzer::FieldManagerBuilder> fmb;  
    Teuchos::RCP<panzer::ResponseLibrary<panzer::Traits> > rLibrary; 
    Teuchos::RCP<panzer::LinearObjFactory<panzer::Traits> > lof;
    Teuchos::RCP<panzer::GlobalData> gd;
  
    buildAssemblyPieces(parameter_on,fmb,rLibrary,gd,lof);

    // Test a transient me
    {
      typedef Thyra::ModelEvaluatorBase MEB;
      typedef Thyra::ModelEvaluatorBase::InArgs<double> InArgs;
      typedef Thyra::ModelEvaluatorBase::OutArgs<double> OutArgs;
      typedef Thyra::VectorBase<double> VectorType;
      typedef Thyra::LinearOpBase<double> OperatorType;
      typedef panzer::ModelEvaluator<double> PME;
      typedef panzer::ExplicitModelEvaluator<double> ExpPME;

      std::vector<Teuchos::RCP<Teuchos::Array<std::string> > > p_names;
      std::vector<Teuchos::RCP<Teuchos::Array<double> > > p_values;
      bool build_transient_support = true;

      Stratimikos::DefaultLinearSolverBuilder builder;
      Teuchos::RCP<Teuchos::ParameterList> validList = Teuchos::rcp(new Teuchos::ParameterList(*builder.getValidParameters()));
      builder.setParameterList(validList);
      RCP<const Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory = builder.createLinearSolveStrategy("Amesos");
    
      RCP<PME> me = Teuchos::rcp(new PME(fmb,rLibrary,lof,p_names,p_values,lowsFactory,gd,build_transient_support,0.0));
      RCP<ExpPME> exp_me = Teuchos::rcp(new ExpPME(me,true,false)); // constant mass, use lumped

      RCP<VectorType> exp_f, f;

      // explicit evaluation
      {
        // set the nominal values
        InArgs nom_vals = exp_me->getNominalValues();
        TEST_ASSERT(nom_vals.supports(MEB::IN_ARG_x));
        TEST_ASSERT(nom_vals.supports(MEB::IN_ARG_x_dot)); // this is supported for stabilization purposes
        TEST_ASSERT(nom_vals.supports(MEB::IN_ARG_alpha)); // alpha and beta support needed for outputting responses
        TEST_ASSERT(nom_vals.supports(MEB::IN_ARG_beta));

        // create in args
        InArgs in_args = exp_me->createInArgs();
        TEST_ASSERT(in_args.supports(MEB::IN_ARG_x));
        TEST_ASSERT(in_args.supports(MEB::IN_ARG_x_dot)); // this is supported for stabilization purposes
        TEST_ASSERT(in_args.supports(MEB::IN_ARG_alpha)); // alpha and beta support needed for outputting responses
        TEST_ASSERT(in_args.supports(MEB::IN_ARG_beta));
        InArgs nomValues = exp_me->getNominalValues();
        RCP<VectorType> x = Thyra::createMember(*exp_me->get_x_space());
        RCP<VectorType> x_dot = Thyra::createMember(*exp_me->get_x_space());
        Thyra::assign(x_dot.ptr(),0.0);
        Thyra::assign(x.ptr(),5.0);
        in_args.set_x(x);
        in_args.set_x_dot(x_dot);
        
        // create out args
        OutArgs out_args = exp_me->createOutArgs();
        TEST_ASSERT(out_args.supports(MEB::OUT_ARG_f));
        TEST_ASSERT(!out_args.supports(MEB::OUT_ARG_W_op));
        TEST_ASSERT(!out_args.supports(MEB::OUT_ARG_W));
        exp_f = Thyra::createMember(*exp_me->get_f_space());
        out_args.set_f(exp_f);

        exp_me->evalModel(in_args, out_args);
      }

      // implicit evaluation
      RCP<OperatorType> mass = me->create_W_op();
      {
        // create in args
        InArgs in_args = me->createInArgs();
        InArgs nomValues = me->getNominalValues();
        RCP<VectorType> x = Thyra::createMember(*me->get_x_space());
        RCP<VectorType> x_dot = Thyra::createMember(*me->get_x_space());
        Thyra::assign(x_dot.ptr(),0.0);
        Thyra::assign(x.ptr(),5.0);
        in_args.set_x(x);
        in_args.set_x_dot(x_dot);
        
        // create out args
        OutArgs out_args = me->createOutArgs();
        f = Thyra::createMember(*me->get_f_space());
        out_args.set_f(f);

        me->evalModel(in_args, out_args);

        in_args.set_x(x);
        in_args.set_x_dot(x_dot);
        in_args.set_alpha(1.0);
        in_args.set_beta(0.0);

        out_args.set_f(Teuchos::null);
        out_args.set_W_op(mass);

        me->setOneTimeDirichletBeta(1.0);
        me->evalModel(in_args, out_args);
      }

      Teuchos::RCP<Thyra::VectorBase<double> > mass_exp_f = Thyra::createMember(*exp_me->get_f_space());
      Thyra::apply(*mass,Thyra::NOTRANS,*exp_f,mass_exp_f.ptr());

      out << "f_i = \n" << Teuchos::describe(*f,Teuchos::VERB_EXTREME) << std::endl;
      out << "f_e = \n" << Teuchos::describe(*mass_exp_f,Teuchos::VERB_EXTREME) << std::endl;

      // it should be that exp_f = -f b/c x_dot=0 in the evaluation
      Thyra::Vp_StV(mass_exp_f.ptr(),1.0,*f);
   
      out << "Error = " << Thyra::norm_2(*mass_exp_f) << std::endl;
      TEST_ASSERT(Thyra::norm_2(*mass_exp_f)<=1e-16);
    }
  }
int main(int argc, char *argv[])
{

  using std::endl;
  typedef double Scalar;
  // typedef double ScalarMag; // unused
  typedef Teuchos::ScalarTraits<Scalar> ST;
  using Teuchos::describe;
  using Teuchos::Array;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::outArg;
  using Teuchos::rcp_implicit_cast;
  using Teuchos::rcp_dynamic_cast;
  using Teuchos::as;
  using Teuchos::ParameterList;
  using Teuchos::CommandLineProcessor;
  typedef Teuchos::ParameterList::PrintOptions PLPrintOptions;
  typedef Thyra::ModelEvaluatorBase MEB;

  bool result, success = true;

  Teuchos::GlobalMPISession mpiSession(&argc,&argv);

  RCP<Epetra_Comm> epetra_comm;
#ifdef HAVE_MPI
  epetra_comm = rcp( new Epetra_MpiComm(MPI_COMM_WORLD) );
#else
  epetra_comm = rcp( new Epetra_SerialComm );
#endif // HAVE_MPI

  RCP<Teuchos::FancyOStream>
    out = Teuchos::VerboseObjectBase::getDefaultOStream();

  try {

    //
    // Read commandline options
    //

    CommandLineProcessor clp;
    clp.throwExceptions(false);
    clp.addOutputSetupOptions(true);

    std::string paramsFileName = "";
    clp.setOption( "params-file", &paramsFileName,
      "File name for XML parameters" );

    double t_final = 1e-3;
    clp.setOption( "final-time", &t_final,
      "Final integration time (initial time is 0.0)" );

    int numTimeSteps = 10;
    clp.setOption( "num-time-steps", &numTimeSteps,
      "Number of (fixed) time steps.  If <= 0.0, then variable time steps are taken" );

    double maxStateError = 1e-14;
    clp.setOption( "max-state-error", &maxStateError,
      "Maximum relative error in the integrated state allowed" );

    Teuchos::EVerbosityLevel verbLevel = Teuchos::VERB_DEFAULT;
    setVerbosityLevelOption( "verb-level", &verbLevel,
      "Top-level verbosity level.  By default, this gets deincremented as you go deeper into numerical objects.",
      &clp );

    Teuchos::EVerbosityLevel solnVerbLevel = Teuchos::VERB_DEFAULT;
    setVerbosityLevelOption( "soln-verb-level", &solnVerbLevel,
      "Solution verbosity level",
      &clp );

    CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv);
    if( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) return parse_return;

    //
    *out << "\nA) Get the base parameter list ...\n";
    //

    RCP<ParameterList>
      paramList = Teuchos::parameterList();
    if (paramsFileName.length())
      updateParametersFromXmlFile( paramsFileName, paramList.ptr() );

    paramList->validateParameters(*getValidParameters());

    const Scalar t_init = 0.0;

    const Rythmos::TimeRange<Scalar> fwdTimeRange(t_init, t_final);
    const Scalar delta_t = t_final / numTimeSteps;
    *out << "\ndelta_t = " << delta_t;

    //
    *out << "\nB) Create the Stratimikos linear solver factory ...\n";
    //
    // This is the linear solve strategy that will be used to solve for the
    // linear system with the W.
    //

    Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
    linearSolverBuilder.setParameterList(sublist(paramList,Stratimikos_name));
    RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> >
      W_factory = createLinearSolveStrategy(linearSolverBuilder);

    //
    *out << "\nC) Create and initalize the forward model ...\n";
    //

    // C.1) Create the underlying EpetraExt::ModelEvaluator

    RCP<EpetraExt::DiagonalTransientModel> epetraStateModel =
      EpetraExt::diagonalTransientModel(
        epetra_comm,
        sublist(paramList,DiagonalTransientModel_name)
        );

    *out <<"\nepetraStateModel valid options:\n";
    epetraStateModel->getValidParameters()->print(
      *out, PLPrintOptions().indent(2).showTypes(true).showDoc(true)
      );

    // C.2) Create the Thyra-wrapped ModelEvaluator

    RCP<Thyra::ModelEvaluator<double> > fwdStateModel =
      epetraModelEvaluator(epetraStateModel, W_factory);

    const RCP<const Thyra::VectorSpaceBase<Scalar> >
      x_space = fwdStateModel->get_x_space();

    const RCP<const Thyra::VectorBase<Scalar> >
      gamma = Thyra::create_Vector(epetraStateModel->get_gamma(), x_space);
    *out << "\ngamma = " << describe(*gamma, solnVerbLevel);

    //
    *out << "\nD) Create the stepper and integrator for the forward problem ...\n";
    //

    RCP<Rythmos::TimeStepNonlinearSolver<double> > fwdTimeStepSolver =
      Rythmos::timeStepNonlinearSolver<double>();
    RCP<Rythmos::StepperBase<Scalar> > fwdStateStepper =
      Rythmos::backwardEulerStepper<double>(fwdStateModel, fwdTimeStepSolver);
    fwdStateStepper->setParameterList(sublist(paramList, RythmosStepper_name));
    RCP<Rythmos::IntegratorBase<Scalar> > fwdStateIntegrator;
    {
      RCP<ParameterList>
        integrationControlPL = sublist(paramList, RythmosIntegrationControl_name);
      integrationControlPL->set( "Take Variable Steps", false );
      integrationControlPL->set( "Fixed dt", as<double>(delta_t) );
      RCP<Rythmos::IntegratorBase<Scalar> >
        defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(
          Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL)
          );
      fwdStateIntegrator = defaultIntegrator;
    }
    fwdStateIntegrator->setParameterList(sublist(paramList, RythmosIntegrator_name));

    //
    *out << "\nE) Solve the forward problem ...\n";
    //

    const MEB::InArgs<Scalar>
      state_ic = fwdStateModel->getNominalValues();
    *out << "\nstate_ic:\n" << describe(state_ic,solnVerbLevel);

    fwdStateStepper->setInitialCondition(state_ic);
    fwdStateIntegrator->setStepper(fwdStateStepper, t_final);

    Array<RCP<const Thyra::VectorBase<Scalar> > > x_final_array;
    fwdStateIntegrator->getFwdPoints(
      Teuchos::tuple<Scalar>(t_final), &x_final_array, NULL, NULL
      );
    const RCP<const Thyra::VectorBase<Scalar> > x_final = x_final_array[0];

    *out << "\nx_final:\n" << describe(*x_final, solnVerbLevel);

    //
    *out << "\nF) Check the solution to the forward problem ...\n";
    //

    const RCP<Thyra::VectorBase<Scalar> >
      x_beta = createMember(x_space),
      x_final_be_exact = createMember(x_space);

    {
      Thyra::ConstDetachedVectorView<Scalar> d_gamma(*gamma);
      Thyra::ConstDetachedVectorView<Scalar> d_x_ic(*state_ic.get_x());
      Thyra::DetachedVectorView<Scalar> d_x_beta(*x_beta);
      Thyra::DetachedVectorView<Scalar> d_x_final_be_exact(*x_final_be_exact);
      const int n = d_gamma.subDim();
      for ( int i = 0; i < n; ++i ) {
        d_x_beta(i) = 1.0 / ( 1.0 - delta_t * d_gamma(i) );
        d_x_final_be_exact(i) = integralPow(d_x_beta(i), numTimeSteps) * d_x_ic(i);
      }
    }

    *out << "\nx_final_be_exact:\n" << describe(*x_final_be_exact, solnVerbLevel);

    result = Thyra::testRelNormDiffErr<Scalar>(
      "x_final", *x_final,
      "x_final_be_exact", *x_final_be_exact,
      "maxStateError", maxStateError,
      "warningTol", 1.0, // Don't warn
      &*out, solnVerbLevel
      );
    if (!result) success = false;

    //
    *out << "\nG) Create the Adjoint ME wrapper object ...\n";
    //

    RCP<Thyra::ModelEvaluator<double> > adjModel =
      Rythmos::adjointModelEvaluator<double>(
        fwdStateModel, fwdTimeRange
        );

    //
    *out << "\nH) Create a stepper and integrator for the adjoint ...\n";
    //

    RCP<Thyra::LinearNonlinearSolver<double> > adjTimeStepSolver =
      Thyra::linearNonlinearSolver<double>();
    RCP<Rythmos::StepperBase<Scalar> > adjStepper =
      Rythmos::backwardEulerStepper<double>(adjModel, adjTimeStepSolver);
    adjStepper->setParameterList(sublist(paramList, RythmosStepper_name));
    RCP<Rythmos::IntegratorBase<Scalar> > adjIntegrator =
      fwdStateIntegrator->cloneIntegrator();

    //
    *out << "\nI) Set up the initial condition for the adjoint at the final time ...\n";
    //

    const RCP<const Thyra::VectorSpaceBase<Scalar> >
      f_space = fwdStateModel->get_f_space();

    // lambda(t_final) = x_final
    const RCP<Thyra::VectorBase<Scalar> > lambda_ic = createMember(f_space);
    V_V( outArg(*lambda_ic), *x_final_be_exact );

    // lambda_dot(t_final,i) = - gamma(i) * lambda(t_final,i)
    const RCP<Thyra::VectorBase<Scalar> > lambda_dot_ic = createMember(f_space);
    Thyra::V_S<Scalar>( outArg(*lambda_dot_ic), ST::zero() );
    Thyra::ele_wise_prod<Scalar>( -ST::one(), *gamma, *lambda_ic,
      outArg(*lambda_dot_ic) );

    MEB::InArgs<Scalar> adj_ic = adjModel->getNominalValues();
    adj_ic.set_x(lambda_ic);
    adj_ic.set_x_dot(lambda_dot_ic);
    *out << "\nadj_ic:\n" << describe(adj_ic,solnVerbLevel);

    //
    *out << "\nJ) Integrate the adjoint backwards in time (using backward time) ...\n";
    //

    adjStepper->setInitialCondition(adj_ic);
    adjIntegrator->setStepper(adjStepper, fwdTimeRange.length());

    Array<RCP<const Thyra::VectorBase<Scalar> > > lambda_final_array;
    adjIntegrator->getFwdPoints(
      Teuchos::tuple<Scalar>(fwdTimeRange.length()), &lambda_final_array, NULL, NULL
      );
    const RCP<const Thyra::VectorBase<Scalar> > lambda_final = lambda_final_array[0];

    *out << "\nlambda_final:\n" << describe(*lambda_final, solnVerbLevel);

    //
    *out << "\nK) Test the final adjoint againt exact discrete solution ...\n";
    //

    {

      const RCP<Thyra::VectorBase<Scalar> >
        lambda_final_be_exact = createMember(x_space);

      {
        Thyra::ConstDetachedVectorView<Scalar> d_gamma(*gamma);
        Thyra::ConstDetachedVectorView<Scalar> d_x_final(*x_final);
        Thyra::DetachedVectorView<Scalar> d_x_beta(*x_beta);
        Thyra::DetachedVectorView<Scalar> d_lambda_final_be_exact(*lambda_final_be_exact);
        const int n = d_gamma.subDim();
        for ( int i = 0; i < n; ++i ) {
          d_lambda_final_be_exact(i) = integralPow(d_x_beta(i), numTimeSteps) * d_x_final(i);
        }
      }

      *out << "\nlambda_final_be_exact:\n" << describe(*lambda_final_be_exact, solnVerbLevel);

      result = Thyra::testRelNormDiffErr<Scalar>(
        "lambda_final", *lambda_final,
        "lambda_final_be_exact", *lambda_final_be_exact,
        "maxStateError", maxStateError,
        "warningTol", 1.0, // Don't warn
        &*out, solnVerbLevel
        );
      if (!result) success = false;

    }

    //
    *out << "\nL) Test the reduced gradient from the adjoint against the discrete forward reduced gradient ...\n";
    //

    {

      const RCP<const Thyra::VectorBase<Scalar> >
        d_d_hat_d_p_from_lambda = lambda_final; // See above

      const RCP<Thyra::VectorBase<Scalar> >
        d_d_hat_d_p_be_exact = createMember(x_space);

      {
        Thyra::ConstDetachedVectorView<Scalar> d_x_ic(*state_ic.get_x());
        Thyra::DetachedVectorView<Scalar> d_x_beta(*x_beta);
        Thyra::DetachedVectorView<Scalar> d_d_d_hat_d_p_be_exact(*d_d_hat_d_p_be_exact);
        const int n = d_x_ic.subDim();
        for ( int i = 0; i < n; ++i ) {
          d_d_d_hat_d_p_be_exact(i) = integralPow(d_x_beta(i), 2*numTimeSteps) * d_x_ic(i);
        }
      }

      *out << "\nd_d_hat_d_p_be_exact:\n" << describe(*d_d_hat_d_p_be_exact, solnVerbLevel);

      result = Thyra::testRelNormDiffErr<Scalar>(
        "d_d_hat_d_p_from_lambda", *d_d_hat_d_p_from_lambda,
        "d_d_hat_d_p_be_exact", *d_d_hat_d_p_be_exact,
        "maxStateError", maxStateError,
        "warningTol", 1.0, // Don't warn
        &*out, solnVerbLevel
        );
      if (!result) success = false;

    }

  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(true,*out,success);

  if(success)
    *out << "\nEnd Result: TEST PASSED" << endl;
  else
    *out << "\nEnd Result: TEST FAILED" << endl;

  return ( success ? 0 : 1 );

} // end main() [Doxygen looks for this!]
  //================================================================
  //================================================================
  // RN_20091215: This needs to be called only once per time step
  // in the beginning to set up the problem.
  //================================================================
  void FC_FUNC(inittrilinos,INITTRILINOS) (int& bandwidth, int& mySize,
	       int* myIndicies, double* myX, double* myY, double* myZ,
	       int* mpi_comm_f) {
// mpi_comm_f: CISM's fortran mpi communicator

#ifdef GLIMMER_MPI
    // Make sure the MPI_Init in Fortran is recognized by C++.
    // We used to call an extra MPI_Init if (!flag), but the behavior of doing so is uncertain,
    // especially if CISM's MPI communicator is a subset of MPI_COMM_WORLD (as can be the case in CESM).
    // Thus, for now, we die with an error message if C++ perceives MPI to be uninitialized.
    // If this causes problems (e.g., if certain MPI implementations seem not to recognize 
    // that MPI has already been initialized), then we will revisit how to handle this.
       int flag;
       MPI_Initialized(&flag);
       if (!flag) {
	 cout << "ERROR in inittrilinos: MPI not initialized according to C++ code" << endl;
	 exit(1);
       }
    MPI_Comm mpi_comm_c = MPI_Comm_f2c(*mpi_comm_f);
    Epetra_MpiComm comm(mpi_comm_c);
    Teuchos::MpiComm<int> tcomm(Teuchos::opaqueWrapper(mpi_comm_c));
#else
    Epetra_SerialComm comm;
    Teuchos::SerialComm<int> tcomm;
#endif

    Teuchos::RCP<const Epetra_Map> rowMap = 
      Teuchos::rcp(new Epetra_Map(-1,mySize,myIndicies,1,comm) );

    TEUCHOS_TEST_FOR_EXCEPTION(!rowMap->UniqueGIDs(), std::logic_error,
       "Error: inittrilinos, myIndices array needs to have Unique entries" 
        << " across all processor.");

    // Diagnostic output for partitioning
    int minSize, maxSize;
    comm.MinAll(&mySize, &minSize, 1);
    comm.MaxAll(&mySize, &maxSize, 1);
    if (comm.MyPID()==0) 
      cout << "\nPartition Info in init_trilinos: Total nodes = " << rowMap->NumGlobalElements()
           << "  Max = " << maxSize << "  Min = " << minSize 
           << "  Ave = " << rowMap->NumGlobalElements() / comm.NumProc() << endl;

    soln = Teuchos::rcp(new Epetra_Vector(*rowMap));

    // Read parameter list once
    try { 
       pl = Teuchos::rcp(new Teuchos::ParameterList("Trilinos Options"));
       Teuchos::updateParametersFromXmlFileAndBroadcast("trilinosOptions.xml", pl.ptr(), tcomm);

       Teuchos::ParameterList validPL("Valid List");;
       validPL.sublist("Stratimikos"); validPL.sublist("Piro");
       pl->validateParameters(validPL, 0);
    }
    catch (std::exception& e) {
      cout << "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n" 
           << e.what() << "\nExiting: Invalid trilinosOptions.xml file."
           << "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" << endl;
      exit(1);
    }
    catch (...) {
      cout << "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n" 
           << "\nExiting: Invalid trilinosOptions.xml file."
           << "\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" << endl;
      exit(1);
    }

    try { 
      // Set the coordinate position of the nodes for ML for repartitioning (important for #procs > 100s)
      if (pl->sublist("Stratimikos").isParameter("Preconditioner Type")) {
         if ("ML" == pl->sublist("Stratimikos").get<string>("Preconditioner Type")) {
           Teuchos::ParameterList& mlList =
              pl->sublist("Stratimikos").sublist("Preconditioner Types").sublist("ML").sublist("ML Settings");
           mlList.set("x-coordinates",myX);
           mlList.set("y-coordinates",myY);
           mlList.set("z-coordinates",myZ);
           mlList.set("PDE equations", 1);
         }
      }

      out = Teuchos::VerboseObjectBase::getDefaultOStream();

      // Reset counters every time step: can remove these lines to have averages over entire run
      linearSolveIters_total = 0;
      linearSolveCount=0;
      linearSolveSuccessCount = 0;

      // Create an interface that holds a CrsMatrix instance and some useful methods.
      interface = Teuchos::rcp(new TrilinosMatrix_Interface(rowMap, bandwidth, comm));

      Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
      linearSolverBuilder.setParameterList(Teuchos::sublist(pl, "Stratimikos"));
      lowsFactory = linearSolverBuilder.createLinearSolveStrategy("");
      lowsFactory->setOStream(out);
      lowsFactory->setVerbLevel(Teuchos::VERB_LOW);

      lows=Teuchos::null;
      thyraOper=Teuchos::null;
    }
    TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
    if (!success) exit(1);
  }
int main(int argc, char *argv[]) {

  int status=0; // 0 = pass, failures are incremented
  int overall_status=0; // 0 = pass, failures are incremented over multiple tests
  bool success=true;

  // Initialize MPI
  Teuchos::GlobalMPISession mpiSession(&argc,&argv);
  int Proc=mpiSession.getRank();
#ifdef HAVE_MPI
  MPI_Comm appComm = MPI_COMM_WORLD;
#else
  int appComm=0;
#endif

  using Teuchos::RCP;
  using Teuchos::rcp;
  std::string inputFile;

  bool doAll = (argc==1);
  if (argc>1) doAll = !strcmp(argv[1],"-v");

  Piro::SolverFactory solverFactory;

#ifdef HAVE_PIRO_RYTHMOS
//  int numTests=4;
  int numTests=3;
#else
  int numTests=2;
#endif

  for (int iTest=0; iTest<numTests; iTest++) {

    if (doAll) {
      switch (iTest) {
       case 0: inputFile="input_Solve_NOX_3.xml"; break;
       case 1: inputFile="input_Solve_LOCA_1.xml"; break;
       case 2: inputFile="input_Solve_Rythmos_2.xml"; break;
// This problem fails in Debug with a throw of "!isFullInitialized".
// Have not successfully debuged this, so disabling. --Andy 5/29/2015
//       case 3: inputFile="input_Solve_RythmosSolver_2.xml"; break;
       default : std::cout << "iTest logic error " << std::endl; exit(-1);
      }
    }
     else {
      inputFile=argv[1];
      iTest = 999;
    }

    if (Proc==0)
     std::cout << "===================================================\n"
          << "======  Running input file "<< iTest <<": "<< inputFile <<"\n"
          << "===================================================\n"
          << std::endl;

    try {

      // Create (1) a Model Evaluator and (2) a ParameterList
      RCP<EpetraExt::ModelEvaluator> epetraModel = rcp(new MockModelEval_A(appComm));

      RCP<Teuchos::ParameterList> piroParams =
         rcp(new Teuchos::ParameterList("Piro Parameters"));
      Teuchos::updateParametersFromXmlFile(inputFile, piroParams.ptr());

      // Use these two objects to construct a Piro solved application
      RCP<const Thyra::ResponseOnlyModelEvaluatorBase<double> > piro;
      {
        const RCP<Teuchos::ParameterList> stratParams = Piro::extractStratimikosParams(piroParams);

        Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
        linearSolverBuilder.setParameterList(stratParams);

        RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory =
          createLinearSolveStrategy(linearSolverBuilder);

        const RCP<Thyra::ModelEvaluator<double> > thyraModel =
          Thyra::epetraModelEvaluator(epetraModel,lowsFactory);

        piro = solverFactory.createSolver(piroParams, thyraModel);
      }

      const Teuchos::RCP<Teuchos::ParameterList> solveParams =
        Teuchos::sublist(Teuchos::sublist(piroParams, "Analysis"), "Solve");

      Teuchos::Array<RCP<const Thyra::VectorBase<double> > > responses;
      Teuchos::Array<Teuchos::Array<RCP<const Thyra::MultiVectorBase<double> > > > sensitivities;
      Piro::PerformSolve(*piro, *solveParams, responses, sensitivities);

      // Extract default input parameters
      const RCP<const Thyra::VectorBase<double> > p1 = piro->getNominalValues().get_p(0);

      // Extract output arguments
      const RCP<const Thyra::VectorBase<double> > g1 = responses[0];
      const RCP<const Thyra::VectorBase<double> > gx = responses[1];
      const RCP<const Thyra::MultiVectorBase<double> > dgdp = sensitivities[0][0];

      // Print out everything
      if (Proc == 0)
        std::cout << "Finished Model Evaluation: Printing everything {Exact in brackets}"
             << "\n-----------------------------------------------------------------"
             << std::setprecision(9) << std::endl;

      std::cout << "\nParameters! {1,1}\n" << *p1 << std::endl;
      std::cout << "\nResponses! {8.0}\n" << *g1 << std::endl;
      std::cout << "\nSolution! {1,2,3,4}\n" << *gx << std::endl;
      if (Teuchos::nonnull(dgdp))
        std::cout <<"\nSensitivities {2.0, -8.0}\n" << *dgdp << std::endl;

      if (Proc == 0)
        std::cout <<
          "\n-----------------------------------------------------------------\n";
    }
    TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
    if (!success) status+=1000;

    overall_status += status;
  }

  if (Proc==0) {
    if (overall_status==0)
      std::cout << "\nTEST PASSED\n" << std::endl;
    else
      std::cout << "\nTEST Failed:  " << overall_status << std::endl;
  }

  return status;
}
int main_(Teuchos::CommandLineProcessor &clp, Xpetra::UnderlyingLib lib, int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>

  using Teuchos::RCP;
  using Teuchos::rcp;

  //
  // MPI initialization
  //

  bool success = false;
  bool verbose = true;
  try {
    RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();

    //
    // Parameters
    //

    Galeri::Xpetra::Parameters<GlobalOrdinal> matrixParameters(clp, 256); // manage parameters of the test case

    std::string xmlFileName = "stratimikos_ParameterList.xml"; clp.setOption("xml",   &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'stratimikos_ParameterList.xml'.");

    switch (clp.parse(argc,argv)) {
      case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED:        return EXIT_SUCCESS; break;
      case Teuchos::CommandLineProcessor::PARSE_ERROR:
      case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
      case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL:                               break;
    }

    // Read in parameter list
    Teuchos::RCP<Teuchos::ParameterList> paramList = Teuchos::getParametersFromXmlFile(xmlFileName);

    //
    // Construct the problem
    //
    RCP<const Map> map = MapFactory::createUniformContigMap(lib, matrixParameters.GetNumGlobalElements(), comm);

    RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
      Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
    RCP<CrsMatrixWrap> A = Pr->BuildMatrix();

    RCP<Vector> X = VectorFactory::Build(map);
    RCP<Vector> B = VectorFactory::Build(map);

    {
      // we set seed for reproducibility
      Utilities::SetRandomSeed(*comm);
      X->randomize();
      A->apply(*X, *B, Teuchos::NO_TRANS, Teuchos::ScalarTraits<Scalar>::one(), Teuchos::ScalarTraits<Scalar>::zero());

      Teuchos::Array<typename Teuchos::ScalarTraits<Scalar>::magnitudeType> norms(1);
      B->norm2(norms);
      B->scale(Teuchos::ScalarTraits<Scalar>::one()/norms[0]);
      X->putScalar(Teuchos::ScalarTraits<Scalar>::zero());
    }

    //
    // Build Thyra linear algebra objects
    //

    RCP<const Thyra::LinearOpBase<Scalar> > thyraA = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyra(A->getCrsMatrix());

    RCP<      Thyra::VectorBase<Scalar> >thyraX = Teuchos::rcp_const_cast<Thyra::VectorBase<Scalar> >(Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyraVector(X));
    RCP<const Thyra::VectorBase<Scalar> >thyraB = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyraVector(B);

    //
    // Build Stratimikos solver
    //

    Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;  // This is the Stratimikos main class (= factory of solver factory).
    Stratimikos::enableMueLu<LocalOrdinal,GlobalOrdinal,Node>(linearSolverBuilder);                // Register MueLu as a Stratimikos preconditioner strategy.
    linearSolverBuilder.setParameterList(paramList);              // Setup solver parameters using a Stratimikos parameter list.

    // Build a new "solver factory" according to the previously specified parameter list.
    RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> > solverFactory = Thyra::createLinearSolveStrategy(linearSolverBuilder);

    // Build a Thyra operator corresponding to A^{-1} computed using the Stratimikos solver.
    Teuchos::RCP<Thyra::LinearOpWithSolveBase<Scalar> > thyraInverseA = Thyra::linearOpWithSolve(*solverFactory, thyraA);

    //
    // Solve Ax = b.
    //

    Thyra::SolveStatus<Scalar> status = Thyra::solve<Scalar>(*thyraInverseA, Thyra::NOTRANS, *thyraB, thyraX.ptr());
    std::cout << status << std::endl;

    success = (status.solveStatus == Thyra::SOLVE_STATUS_CONVERGED);
  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);

  return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}