Exemple #1
0
int main(int argc, char *argv[])
{
  MPI_Init(&argc,&argv);
  Epetra_MpiComm Comm(MPI_COMM_WORLD);
  Epetra_MpiComm SerialComm(MPI_COMM_SELF);

  if (Comm.MyPID() == 0) {

    ParameterList GaleriList;
    GaleriList.set("nx", 10);
    GaleriList.set("ny", 10);
    GaleriList.set("mx", 1);
    GaleriList.set("my", 1);

    Epetra_Map* Map = CreateMap("Cartesian2D", SerialComm, GaleriList);
    Epetra_CrsMatrix* A = CreateCrsMatrix("Laplace2D", Map, GaleriList);

    Epetra_Vector LHS(A->OperatorDomainMap());
    Epetra_Vector RHS(A->OperatorRangeMap());

    LHS.PutScalar(0.0);
    RHS.Random();

    Epetra_LinearProblem problem(A, &LHS, &RHS);

    AztecOO solver(problem);

    ParameterList MLList;

    ML_Epetra::SetDefaults("SA",MLList);
    MLList.set("ML output", 0);

    ML_Epetra::MultiLevelPreconditioner* MLPrec =
      new ML_Epetra::MultiLevelPreconditioner(*A, MLList);

    solver.SetPrecOperator(MLPrec);
    solver.SetAztecOption(AZ_solver, AZ_cg_condnum);
    solver.SetAztecOption(AZ_output, 32);
    solver.Iterate(500, 1e-12);

    // destroy the preconditioner
    delete MLPrec;

    delete A;
    delete Map;
  }

  MPI_Finalize();
  return(0);

} // main driver
  /*! \brief Set up validators specific to this Problem
  */
  static void getValidParameters(ParameterList & pl)
  {
    RCP<Teuchos::StringValidator> order_method_Validator =
      Teuchos::rcp( new Teuchos::StringValidator(
        Teuchos::tuple<std::string>( "rcm", "minimum_degree", "natural",
          "random", "sorted_degree", "scotch", "nd" )));
    pl.set("order_method", "rcm", "order algorithm",
      order_method_Validator);

    RCP<Teuchos::StringValidator> order_package_Validator = Teuchos::rcp(
      new Teuchos::StringValidator(
        Teuchos::tuple<std::string>( "amd", "package2", "package3" )));
    pl.set("order_package", "amd", "package to use in ordering",
      order_package_Validator);
  }
Exemple #3
0
 ParameterList TPINode::getDefaultParameters() 
 {
   ParameterList params;
   params.set("Verbose",     0);
   params.set("Num Threads", 0);
   return params;
 }
 /*! \brief Set up validators specific to this algorithm
   */
 static void getValidParameters(ParameterList & pl)
 {
   RCP<Teuchos::EnhancedNumberValidator<int>> forTestingOnlyFlag_Validator =
     Teuchos::rcp( new Teuchos::EnhancedNumberValidator<int>(0, 1000, 1, 0) );
   pl.set("forTestingOnlyFlag", 0, "Used only for testing; look at "
     "Zoltan2_AlgForTestingOnly for interpretations",
     forTestingOnlyFlag_Validator);
 }
 /*! \brief Set up validators specific to this algorithm
 */
 static void getValidParameters(ParameterList & pl)
 {
   RCP<Teuchos::StringValidator> color_choice_Validator = Teuchos::rcp(
     new Teuchos::StringValidator(
     Teuchos::tuple<std::string>(
       "FirstFit", "Random", "RandomFast", "LeastUsed" )));
   pl.set("color_choice", "FirstFit", "selection criterion for coloring",
     color_choice_Validator);
 }
  RCP<ParameterList> HybridPlatform::listSupportedNodes()
  {
    RCP<ParameterList> list = Teuchos::parameterList();
    {
      ParameterList subpl;
      subpl.set("NodeType","KokkosClassic::SerialNode");
      subpl.setParameters( KokkosClassic::SerialNode::getDefaultParameters() );
      list->set("=-1",subpl);
    }
#ifdef HAVE_KOKKOSCLASSIC_TBB
    {
      ParameterList subpl;
      subpl.set("NodeType","KokkosClassic::TBBNode");
      subpl.setParameters( KokkosClassic::TBBNode::getDefaultParameters() );
      list->set("=-2",subpl);
    }
#endif        
#ifdef HAVE_KOKKOSCLASSIC_OPENMP
    {
      ParameterList subpl;
      subpl.set("NodeType","KokkosClassic::OpenMPNode");
      subpl.setParameters( KokkosClassic::OpenMPNode::getDefaultParameters() );
      list->set("=-3",subpl);
    }
#endif        
#ifdef HAVE_KOKKOSCLASSIC_THREADPOOL
    {
      ParameterList subpl;
      subpl.set("NodeType","KokkosClassic::TPINode");
      subpl.setParameters( KokkosClassic::TPINode::getDefaultParameters() );
      list->set("=-4",subpl);
    }
#endif        
#ifdef HAVE_KOKKOSCLASSIC_THRUST
    {
      ParameterList subpl;
      subpl.set("NodeType","KokkosClassic::ThrustGPUNode");
      subpl.setParameters( KokkosClassic::ThrustGPUNode::getDefaultParameters() );
      list->set("=-5",subpl);
    }
#endif        
    return list;
  }
/**
 * Test the ArrayLengthDependency.
 */
TEUCHOS_UNIT_TEST(Teuchos_Dependencies, testArrayLengthDep){
	RCP<ParameterList> My_deplist = RCP<ParameterList>(new ParameterList);
	RCP<DependencySheet> depSheet1 =
    RCP<DependencySheet>(new DependencySheet);

	ParameterList
	numberArrayLengthDepList = My_deplist->sublist(
    "Number Array Length Dependency List", false,
    "Number Array Length Dependecy testing list.");
	numberArrayLengthDepList.set("Array Length", 10, "array length setter");
	Array<double> variableLengthArray(11,23.0);
	RCP<EnhancedNumberValidator<double> >
	varLengthArrayVali = RCP<EnhancedNumberValidator<double> >(
  		new EnhancedNumberValidator<double>(10,50,4)
	);
	numberArrayLengthDepList.set(
    "Variable Length Array", variableLengthArray, "variable length array",
	  RCP<ArrayNumberValidator<double> >(
      new ArrayNumberValidator<double>(varLengthArrayVali)));

	RCP<NumberArrayLengthDependency<int, double> >
	  arrayLengthDep(
  		new NumberArrayLengthDependency<int, double>(
			numberArrayLengthDepList.getEntryRCP("Array Length"),
			numberArrayLengthDepList.getEntryRCP("Variable Length Array"),
      rcp(new AdditionFunction<int>(1))
		)
	);
	depSheet1->addDependency(arrayLengthDep);
  Array<double> curArray =
    numberArrayLengthDepList.get<Array<double> >("Variable Length Array");
	TEST_ASSERT(curArray.length() ==11);
	numberArrayLengthDepList.set("Array Length", 12);
	arrayLengthDep()->evaluate();
  curArray =
    numberArrayLengthDepList.get<Array<double> >("Variable Length Array");
  out << curArray.length() << std::endl;
	TEST_ASSERT(curArray.length() ==13);
	numberArrayLengthDepList.set("Array Length", -2);
	TEST_THROW(arrayLengthDep()->evaluate(),
    Exceptions::InvalidParameterValue);
}
/**
 * Test the TwoDColDependency.
 */
TEUCHOS_UNIT_TEST(Teuchos_Dependencies, testTwoDColDependency){
	RCP<ParameterList> My_deplist = RCP<ParameterList>(new ParameterList);
	RCP<DependencySheet> depSheet1 =
    RCP<DependencySheet>(new DependencySheet);

	ParameterList
	colNumDepList = My_deplist->sublist(
    "2D Col Depdency List", false,
    "2D Col Dependecy testing list.");
	colNumDepList.set("Num cols", 2, "num cols setter");
  TwoDArray<double> variableColsArray(11,3,16.5);
	RCP<EnhancedNumberValidator<double> >
	varColArrayVali = RCP<EnhancedNumberValidator<double> >(
  		new EnhancedNumberValidator<double>(10,50,4)
	);
	colNumDepList.set(
    "Variable Col Array", variableColsArray, "variable col array",
	  RCP<TwoDArrayNumberValidator<double> >(
      new TwoDArrayNumberValidator<double>(varColArrayVali)));

	RCP<TwoDColDependency<int, double> >
	  arrayColDep = rcp(
  		new TwoDColDependency<int, double>(
		  colNumDepList.getEntryRCP("Num cols"),
			colNumDepList.getEntryRCP("Variable Col Array") ,
      rcp(new AdditionFunction<int>(1))
		)
	);
	depSheet1->addDependency(arrayColDep);
  TwoDArray<double> curArray =
    colNumDepList.get<TwoDArray<double> >("Variable Col Array");
	TEST_EQUALITY_CONST(curArray.getNumCols(),3);
	colNumDepList.set("Num cols", 4);
	arrayColDep()->evaluate();
  curArray =
    colNumDepList.get<TwoDArray<double> >("Variable Col Array");
	TEST_EQUALITY_CONST(curArray.getNumCols(),5);
	colNumDepList.set("Num cols", -2);
	TEST_THROW(arrayColDep()->evaluate(),
    Exceptions::InvalidParameterValue);
}
TEUCHOS_UNIT_TEST_TEMPLATE_1_DECL(Teuchos_Validator, TwoDArrayNumberValidatorConverterTest, T)
{
  std::string arrayParameterName = "array";
  ParameterList myList;

  const T arrayValidatorLen = as<T>(11);
  RCP<TwoDArrayNumberValidator< T > > arrayValidator =
    rcp(new TwoDArrayNumberValidator< T >(
      rcp(new EnhancedNumberValidator<T>(as<T>(0), arrayValidatorLen))));
  myList.set(arrayParameterName,
    TwoDArray< T >(4,4, 10), "array parameter", arrayValidator);

  RCP<ParameterList> readInPL = writeThenReadPL(myList);

  RCP<const EnhancedNumberValidator< T > > readInPrototypeValidator =
    rcp_dynamic_cast<const TwoDArrayValidator<EnhancedNumberValidator<T>, T > >(
      readInPL->getEntry(
        arrayParameterName).validator(), true)->getPrototype();
  RCP<const EnhancedNumberValidator< T > > actualPrototypeValidator =
    arrayValidator->getPrototype();

  TEST_EQUALITY(
    readInPrototypeValidator->getMin(),
    actualPrototypeValidator->getMin()
  );
  TEST_EQUALITY(
    readInPrototypeValidator->getMax(),
    actualPrototypeValidator->getMax()
  );
  TEST_EQUALITY(
    readInPrototypeValidator->getStep(),
    actualPrototypeValidator->getStep()
  );
  TEST_EQUALITY(
    readInPrototypeValidator->getPrecision(),
    actualPrototypeValidator->getPrecision()
  );
  TEST_EQUALITY(
    readInPrototypeValidator->hasMin(),
    actualPrototypeValidator->hasMin()
  );
  TEST_EQUALITY(
    readInPrototypeValidator->hasMax(),
    actualPrototypeValidator->hasMax()
  );
}
Exemple #10
0
  void RebalanceAcFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level &fineLevel, Level &coarseLevel) const {
    FactoryMonitor m(*this, "Computing Ac", coarseLevel);

    RCP<Matrix> originalAc = Get< RCP<Matrix> >(coarseLevel, "A");

    RCP<const Import> rebalanceImporter = Get< RCP<const Import> >(coarseLevel, "Importer");

    if (rebalanceImporter != Teuchos::null) {
      RCP<Matrix> rebalancedAc;
      {
        SubFactoryMonitor subM(*this, "Rebalancing existing Ac", coarseLevel);
        RCP<const Map> targetMap = rebalanceImporter->getTargetMap();

        const ParameterList & pL = GetParameterList();

        ParameterList XpetraList;
        if (pL.get<bool>("useSubcomm") == true) {
          GetOStream(Runtime0,0) << "Replacing maps with a subcommunicator" << std::endl;
          XpetraList.set("Restrict Communicator",true);
        }
        // NOTE: If the communicator is restricted away, Build returns Teuchos::null.
        rebalancedAc = MatrixFactory::Build(originalAc, *rebalanceImporter, targetMap, targetMap, rcp(&XpetraList,false));

        if (!rebalancedAc.is_null())
          rebalancedAc->SetFixedBlockSize(originalAc->GetFixedBlockSize());

        Set(coarseLevel, "A", rebalancedAc);
      }

      if (!rebalancedAc.is_null()) {
        RCP<ParameterList> params = rcp(new ParameterList());
        params->set("printLoadBalancingInfo", true);
        GetOStream(Statistics0, 0) << Utils::PrintMatrixInfo(*rebalancedAc, "Ac (rebalanced)", params);
      }

    } else {
      // Ac already built by the load balancing process and no load balancing needed
      GetOStream(Warnings0, 0) << "No rebalancing" << std::endl;
      GetOStream(Warnings0, 0) << "Jamming A into Level " << coarseLevel.GetLevelID() << " w/ generating factory "
                               << this << std::endl;
      Set(coarseLevel, "A", originalAc);
    }

  } //Build()
int main(int argc, char** argv)
{
  typedef int                                                       Ordinal;
  typedef double                                                    Scalar;

  typedef Tpetra::MpiPlatform<KokkosClassic::DefaultNode::DefaultNodeType> MpiPlatform;
  typedef Tpetra::SerialPlatform<KokkosClassic::DefaultNode::DefaultNodeType> SerialPlatform;

  typedef MpiPlatform::NodeType                                     MpiNodeType;
  typedef SerialPlatform::NodeType                                  SerialNodeType;

  using namespace Teuchos;

  oblackholestream blackhole;
  GlobalMPISession mpiSession(&argc,&argv,&blackhole);

  ParameterList pl;

  Ordinal numThreads=1;
  pl.set("Num Threads",numThreads);
  RCP<MpiNodeType> mpiNode = rcp(new MpiNodeType(pl));
  RCP<SerialNodeType> serialNode = rcp(new SerialNodeType(pl));

  MpiPlatform    myMpiPlat(mpiNode);
  SerialPlatform mySerialPlat(serialNode);

  {
    RCP<const Comm<int> > teuchosComm = mySerialPlat.getComm();
    RCP<const Epetra_Comm> epetraComm = Teuchos2Epetra_Comm(teuchosComm);

    assert(epetraComm != Teuchos::null);
  }

  {
    RCP<const Comm<int> > teuchosComm = myMpiPlat.getComm();
    RCP<const Epetra_Comm> epetraComm = Teuchos2Epetra_Comm(teuchosComm);

    assert(epetraComm != Teuchos::null);
  }

  return(0);
} //main
int main(int argc, char *argv[]) {

#ifdef HAVE_MPI
  MPI_Init(&argc, &argv);
  Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
  Epetra_SerialComm Comm;
#endif

  // initialize the random number generator

  int ml_one = 1;
  ML_srandom1(&ml_one);
  // ===================== //
  // create linear problem //
  // ===================== //

  ParameterList GaleriList;
  GaleriList.set("nx", 10);
  GaleriList.set("ny", 10);
  GaleriList.set("nz", 10 * Comm.NumProc());
  GaleriList.set("mx", 1);
  GaleriList.set("my", 1);
  GaleriList.set("mz", Comm.NumProc());

  Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
  Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);
  Epetra_MultiVector* Coords = CreateCartesianCoordinates("3D",Map,GaleriList);

  Epetra_Vector LHS(*Map);
  Epetra_Vector RHS(*Map);

  Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);

  Teuchos::ParameterList MLList;
  double TotalErrorResidual = 0.0, TotalErrorExactSol = 0.0;

  // ====================== //
  // default options for SA //
  // ====================== //

  if (Comm.MyPID() == 0) PrintLine();

  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "Gauss-Seidel");
  char mystring[80];
  strcpy(mystring,"SA");
  TestMultiLevelPreconditioner(mystring, MLList, Problem,
                               TotalErrorResidual, TotalErrorExactSol);


  // ============================================== //
  // default options for SA, efficient symmetric GS //
  // ============================================== //

  if (Comm.MyPID() == 0) PrintLine();

  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "Gauss-Seidel");
  MLList.set("smoother: Gauss-Seidel efficient symmetric",true);

  TestMultiLevelPreconditioner(mystring, MLList, Problem,
                               TotalErrorResidual, TotalErrorExactSol,true);

  // ============================== //
  // default options for SA, Jacobi //
  // ============================== //

  if (Comm.MyPID() == 0) PrintLine();

  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "Jacobi");

  TestMultiLevelPreconditioner(mystring, MLList, Problem, TotalErrorResidual,
                               TotalErrorExactSol,true);

  // =========================== //
  // default options for SA, Cheby //
  // =========================== //

  if (Comm.MyPID() == 0) PrintLine();

  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "Chebyshev");

  TestMultiLevelPreconditioner(mystring, MLList, Problem,
                               TotalErrorResidual, TotalErrorExactSol);



  // =========================== //
  // Specifying Ifpack coarse lists correctly
  // =========================== //
#ifdef HAVE_ML_IFPACK
  if (Comm.MyPID() == 0) PrintLine();
  ML_Epetra::SetDefaults("SA",MLList);

  if(!Comm.MyPID()) {
    MLList.set("ML print initial list",1);
    MLList.set("ML print final list",1);
  }

  MLList.set("smoother: type","ILU");
  MLList.set("coarse: type","ILUT");
  ParameterList &fList = MLList.sublist("smoother: ifpack list");
  fList.set("fact: level-of-fill",1);
  ParameterList &cList = MLList.sublist("coarse: ifpack list");
  cList.set("fact: ilut level-of-fill",1e-2);
  TestMultiLevelPreconditioner(mystring, MLList, Problem,
                               TotalErrorResidual, TotalErrorExactSol);
#endif


  // =========================== //
  // Specifying level sublists
  // =========================== //
  if (Comm.MyPID() == 0) PrintLine();
  ParameterList LevelList;
  ML_Epetra::SetDefaults("SA",LevelList);
  ParameterList &smList = LevelList.sublist("smoother: list (level 0)");
  smList.set("smoother: type","Jacobi");
  smList.set("smoother: sweeps",5);
  ParameterList &smList2 = LevelList.sublist("smoother: list (level 1)");
  smList2.set("smoother: type","symmetric Gauss-Seidel");
  smList2.set("smoother: sweeps",3);
  ParameterList &coarseList = LevelList.sublist("coarse: list");
  coarseList.set("smoother: type","symmetric Gauss-Seidel");
  TestMultiLevelPreconditioner(mystring, LevelList, Problem,
                               TotalErrorResidual, TotalErrorExactSol);


  // =========================== //
  // Ifpack G-S w/ L1
  // =========================== //
#ifdef HAVE_ML_IFPACK
  if (Comm.MyPID() == 0) PrintLine();
  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: use l1 Gauss-Seidel",true);
  MLList.set("smoother: type", "Gauss-Seidel");
  TestMultiLevelPreconditioner(mystring, MLList, Problem,
                               TotalErrorResidual, TotalErrorExactSol);
#endif

  // =========================== //
  // Ifpack SGS w/ L1
  // =========================== //
#ifdef HAVE_ML_IFPACK
  if (Comm.MyPID() == 0) PrintLine();
  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: use l1 Gauss-Seidel",true);
  MLList.set("smoother: type", "symmetric Gauss-Seidel");
  TestMultiLevelPreconditioner(mystring, MLList, Problem,
                               TotalErrorResidual, TotalErrorExactSol);
#endif


  // =========================== //
  // Autodetected Line SGS (trivial lines) 
  // =========================== //
  if (Comm.MyPID() == 0) PrintLine();
  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "line Gauss-Seidel");
  MLList.set("smoother: line detection threshold",0.1);
  MLList.set("x-coordinates",(*Coords)[0]);
  MLList.set("y-coordinates",(*Coords)[1]);
  MLList.set("z-coordinates",(*Coords)[2]);
  TestMultiLevelPreconditioner(mystring, MLList, Problem,
                               TotalErrorResidual, TotalErrorExactSol);
  

  // ===================== //
  // print out total error //
  // ===================== //

  if (Comm.MyPID() == 0) {
    cout << endl;
    cout << "......Total error for residual        = " << TotalErrorResidual << endl;
    cout << "......Total error for exact solution  = " << TotalErrorExactSol << endl;
    cout << endl;
  }

  delete Matrix;
  delete Coords;
  delete Map;


  if (TotalErrorResidual > 1e-8) {
    cerr << "Error: `MultiLevelPrecoditioner_Sym.exe' failed!" << endl;
    exit(EXIT_FAILURE);
  }

#ifdef HAVE_MPI
  MPI_Finalize();
#endif

  if (Comm.MyPID() == 0)
    cerr << "`MultiLevelPrecoditioner_Sym.exe' passed!" << endl;

  return (EXIT_SUCCESS);
}
Exemple #13
0
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"

  using Teuchos::RCP; using Teuchos::rcp;
  using Teuchos::TimeMonitor;
  //using Galeri::Xpetra::CreateCartesianCoordinates;

  Teuchos::oblackholestream blackhole;
  Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole);

  RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
  RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
  out->setOutputToRootOnly(0);
  *out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;

  // out->setOutputToRootOnly(-1);
  // out->precision(12);

  //FIXME we need a HAVE_MUELU_LONG_LONG_INT option
  //#ifndef HAVE_TEUCHOS_LONG_LONG_INT
  *out << "Warning: scaling test was not compiled with long long int support" << std::endl;
  //#endif

  //
  // SET TEST PARAMETERS
  //
  // Note: use --help to list available options.
  Teuchos::CommandLineProcessor clp(false);

  // Default is Laplace1D with nx = 8748.
  // It's a nice size for 1D and perfect aggregation. (6561 = 3^8)
  //Nice size for 1D and perfect aggregation on small numbers of processors. (8748 = 4*3^7)
  Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
  Xpetra::Parameters xpetraParameters(clp);                   // manage parameters of xpetra

  // Custom command line parameters
  // - Debug
  int optDebug   = 0;                     clp.setOption("debug",          &optDebug,              "pause to attach debugger");
  int optDump    = 0;                     clp.setOption("dump",           &optDump,               "write matrix to file");
  int optTimings = 0;                     clp.setOption("timings",        &optTimings,            "print timings to screen");

  // - Levels
  LO  optMaxLevels     = 10;              clp.setOption("maxLevels",      &optMaxLevels,          "maximum number of levels allowed");
  int optMaxCoarseSize = 50;              clp.setOption("maxCoarseSize",  &optMaxCoarseSize,      "maximum #dofs in coarse operator"); //FIXME clp doesn't like long long int

  // - Smoothed-Aggregation
  Scalar optSaDamping = 4./3;             clp.setOption("saDamping",      &optSaDamping,          "prolongator damping factor");

  // - Aggregation
  std::string optAggOrdering = "natural"; clp.setOption("aggOrdering",    &optAggOrdering,        "aggregation ordering strategy (natural, random, graph)");
  int optMinPerAgg = 2;                   clp.setOption("minPerAgg",      &optMinPerAgg,          "minimum #DOFs per aggregate");
  int optMaxNbrSel = 0;                   clp.setOption("maxNbrSel",      &optMaxNbrSel,          "maximum # of nbrs allowed to be in other aggregates");

  // - R
  int optExplicitR = 1;                   clp.setOption("explicitR",      &optExplicitR,          "restriction will be explicitly stored as transpose of prolongator");

  // - Smoothers
  std::string optSmooType = "sgs";        clp.setOption("smooType",       &optSmooType,           "smoother type ('l1-sgs', 'sgs 'or 'cheby')");
  int optSweeps = 2;                      clp.setOption("sweeps",         &optSweeps,             "sweeps to be used in SGS (or Chebyshev degree)");

  // - Repartitioning
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
  int optRepartition = 1;                 clp.setOption("repartition",    &optRepartition,        "enable repartitioning (0=no repartitioning, 1=Zoltan RCB, 2=Isorropia+Zoltan PHG");
  LO optMinRowsPerProc = 2000;            clp.setOption("minRowsPerProc", &optMinRowsPerProc,     "min #rows allowable per proc before repartitioning occurs");
  double optNnzImbalance = 1.2;           clp.setOption("nnzImbalance",   &optNnzImbalance,       "max allowable nonzero imbalance before repartitioning occurs");
#else
  int optRepartition = 0;
#endif // HAVE_MPI && HAVE_MUELU_ZOLTAN

  // - Solve
  int    optFixPoint = 1;                 clp.setOption("fixPoint",       &optFixPoint,           "apply multigrid as solver");
  int    optPrecond  = 1;                 clp.setOption("precond",        &optPrecond,            "apply multigrid as preconditioner");
  LO     optIts      = 10;                clp.setOption("its",            &optIts,                "number of multigrid cycles");
  double optTol      = 1e-7;              clp.setOption("tol",            &optTol,                "stopping tolerance for Krylov method");

  switch (clp.parse(argc, argv)) {
  case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED:        return EXIT_SUCCESS; break;
  case Teuchos::CommandLineProcessor::PARSE_ERROR:
  case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
  case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL:                               break;
  }

  RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));

  if (optDebug) {
    Utils::PauseForDebugger();
  }

  matrixParameters.check();
  xpetraParameters.check();
  // TODO: check custom parameters
  std::transform(optSmooType.begin(), optSmooType.end(), optSmooType.begin(), ::tolower);
  Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();

  if (comm->getRank() == 0) {
    std::cout << xpetraParameters << matrixParameters;
    // TODO: print custom parameters // Or use paramList::print()!
  }

  //
  // CREATE INITIAL MATRIX                                                          */
  //
  RCP<const Map> map;
  RCP<Matrix> A;

  RCP<MultiVector> coordinates;
  {
    TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build"));

    map = MapFactory::Build(lib, matrixParameters.GetNumGlobalElements(), 0, comm);
    Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
        Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
    A = Pr->BuildMatrix();

    if (matrixParameters.GetMatrixType() == "Laplace1D") {
      coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
    }
    else if (matrixParameters.GetMatrixType() == "Laplace2D") {
      coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
    }
    else if (matrixParameters.GetMatrixType() == "Laplace3D") {
      coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
    }
  }

  //
  //
  //

  // dump matrix to file
  if (optDump) {
    std::string fileName = "Amat.mm";
    Utils::Write(fileName, *A);
  }

  RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, 1);
  nullspace->putScalar( (SC) 1.0);
  Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);

  nullspace->norm1(norms);
  if (comm->getRank() == 0)
    std::cout << "||NS|| = " << norms[0] << std::endl;

  RCP<MueLu::Hierarchy<SC, LO, GO, NO, LMO> > H;

  //
  //
  // SETUP
  //
  //

  {
    TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup"));

    //
    // Hierarchy
    //

    H = rcp(new Hierarchy());
    H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
    H->SetMaxCoarseSize((GO) optMaxCoarseSize);

    //
    // Finest level
    //

    RCP<Level> Finest = H->GetLevel();
    Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
    Finest->Set("A",           A);
    Finest->Set("Nullspace",   nullspace);
    Finest->Set("Coordinates", coordinates); //FIXME: XCoordinates, YCoordinates, ..

    //
    // FactoryManager
    //

    FactoryManager M;

    //
    //
    // Aggregation
    //

    {
      RCP<UncoupledAggregationFactory> AggregationFact = rcp(new UncoupledAggregationFactory());
      *out << "========================= Aggregate option summary =========================" << std::endl;
      *out << "min DOFs per aggregate :                " << optMinPerAgg << std::endl;
      *out << "min # of root nbrs already aggregated : " << optMaxNbrSel << std::endl;
      AggregationFact->SetMinNodesPerAggregate(optMinPerAgg);  //TODO should increase if run anything othpermRFacter than 1D
      AggregationFact->SetMaxNeighAlreadySelected(optMaxNbrSel);
      std::transform(optAggOrdering.begin(), optAggOrdering.end(), optAggOrdering.begin(), ::tolower);
      if (optAggOrdering == "natural") {
        *out << "aggregate ordering :                    NATURAL" << std::endl;
        AggregationFact->SetOrdering(MueLu::AggOptions::NATURAL);
      } else if (optAggOrdering == "random") {
        *out << "aggregate ordering :                    RANDOM" << std::endl;
        AggregationFact->SetOrdering(MueLu::AggOptions::RANDOM);
      } else if (optAggOrdering == "graph") {
        *out << "aggregate ordering :                    GRAPH" << std::endl;
        AggregationFact->SetOrdering(MueLu::AggOptions::GRAPH);
      } else {
        std::string msg = "main: bad aggregation option """ + optAggOrdering + """.";
        throw(MueLu::Exceptions::RuntimeError(msg));
      }
      //AggregationFact->SetPhase3AggCreation(0.5);
      M.SetFactory("Aggregates", AggregationFact);

    *out << "=============================================================================" << std::endl;
    }

    //
    // Transfer
    //

    {
      //
      // Non rebalanced factories
      //

      RCP<SaPFactory> PFact = rcp(new SaPFactory());
      PFact->SetDampingFactor(optSaDamping);

      RCP<Factory>    RFact = rcp(new TransPFactory());

      RCP<RAPFactory> AFact = rcp(new RAPFactory());
      AFact->setVerbLevel(Teuchos::VERB_HIGH);
      if (!optExplicitR) {
        H->SetImplicitTranspose(true);
        ParameterList Aclist = *(AFact->GetValidParameterList());
        Aclist.set("implicit transpose", true);
        AFact->SetParameterList(Aclist);
        if (comm->getRank() == 0) std::cout << "\n\n* ***** USING IMPLICIT RESTRICTION OPERATOR ***** *\n" << std::endl;
      }

      //
      // Repartitioning (if needed)
      //

      if (optRepartition == 0) {
        // No repartitioning

        // Configure FactoryManager
        M.SetFactory("P", PFact);
        M.SetFactory("R", RFact);
        M.SetFactory("A", AFact);

      } else {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
        // Repartitioning

        // The Factory Manager will be configured to return the rebalanced versions of P, R, A by default.
        // Everytime we want to use the non-rebalanced versions, we need to explicitly define the generating factory.
        RFact->SetFactory("P", PFact);
        //
        AFact->SetFactory("P", PFact);
        AFact->SetFactory("R", RFact);

        // Transfer coordinates
        RCP<CoordinatesTransferFactory> TransferCoordinatesFact = rcp(new CoordinatesTransferFactory());
        AFact->AddTransferFactory(TransferCoordinatesFact); // FIXME REMOVE

        // Compute partition (creates "Partition" object)
        if(optRepartition == 1) { // use plain Zoltan Interface

        } else if (optRepartition == 2) { // use Isorropia + Zoltan interface

        }

        // Repartitioning (creates "Importer" from "Partition")
        RCP<Factory> RepartitionFact = rcp(new RepartitionFactory());
        {
          Teuchos::ParameterList paramList;
          paramList.set("minRowsPerProcessor", optMinRowsPerProc);
          paramList.set("nonzeroImbalance", optNnzImbalance);
          RepartitionFact->SetParameterList(paramList);
        }
        RepartitionFact->SetFactory("A", AFact);

        if(optRepartition == 1) {
          RCP<Factory> ZoltanFact = rcp(new ZoltanInterface());
          ZoltanFact->SetFactory("A", AFact);
          ZoltanFact->SetFactory("Coordinates", TransferCoordinatesFact);
          RepartitionFact->SetFactory("Partition", ZoltanFact);
        }
        else if(optRepartition == 2) {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ISORROPIA)
          RCP<MueLu::IsorropiaInterface<LO, GO, NO, LMO> > isoInterface = rcp(new MueLu::IsorropiaInterface<LO, GO, NO, LMO>());
          isoInterface->SetFactory("A", AFact);
          // we don't need Coordinates here!
          RepartitionFact->SetFactory("Partition", isoInterface);
#else
          if (comm->getRank() == 0)
            std::cout << "Please recompile Trilinos with Isorropia support enabled." << std::endl;
          return EXIT_FAILURE;
#endif
        }


        // Reordering of the transfer operators
        RCP<Factory> RebalancedPFact = rcp(new RebalanceTransferFactory());
        RebalancedPFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Interpolation")));
        RebalancedPFact->SetFactory("P", PFact);
        RebalancedPFact->SetFactory("Coordinates", TransferCoordinatesFact);
        RebalancedPFact->SetFactory("Nullspace", M.GetFactory("Ptent")); // TODO

        RCP<Factory> RebalancedRFact = rcp(new RebalanceTransferFactory());
        RebalancedRFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Restriction")));
        RebalancedRFact->SetFactory("R", RFact);

        // Compute Ac from rebalanced P and R
        RCP<Factory> RebalancedAFact = rcp(new RebalanceAcFactory());
        RebalancedAFact->SetFactory("A", AFact);

        // Configure FactoryManager
        M.SetFactory("A", RebalancedAFact);
        M.SetFactory("P", RebalancedPFact);
        M.SetFactory("R", RebalancedRFact);
        M.SetFactory("Nullspace",   RebalancedPFact);
        M.SetFactory("Coordinates", RebalancedPFact);
        M.SetFactory("Importer",    RepartitionFact);

#else
        TEUCHOS_TEST_FOR_EXCEPT(true);
#endif
      } // optRepartition

    } // Transfer

    //
    // Smoothers
    //

    {
      std::string ifpackType;
      Teuchos::ParameterList ifpackList;
      ifpackList.set("relaxation: sweeps", (LO) optSweeps);
      ifpackList.set("relaxation: damping factor", (SC) 1.0);
      if (optSmooType == "sgs") {
        ifpackType = "RELAXATION";
        ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
      }
      else if (optSmooType == "l1-sgs") {
        ifpackType = "RELAXATION";
        ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
        ifpackList.set("relaxation: use l1", true);
      } else if (optSmooType == "cheby") {
        ifpackType = "CHEBYSHEV";
        ifpackList.set("chebyshev: degree", (LO) optSweeps);

        if (matrixParameters.GetMatrixType() == "Laplace1D") {
          ifpackList.set("chebyshev: ratio eigenvalue", (SC) 3);
        }
        else if (matrixParameters.GetMatrixType() == "Laplace2D") {
          ifpackList.set("chebyshev: ratio eigenvalue", (SC) 7);
        }
        else if (matrixParameters.GetMatrixType() == "Laplace3D") {
          ifpackList.set("chebyshev: ratio eigenvalue", (SC) 20);
        }
        // ifpackList.set("chebyshev: max eigenvalue", (double) -1.0);
        // ifpackList.set("chebyshev: min eigenvalue", (double) 1.0);
      }

      RCP<SmootherPrototype> smootherPrototype = rcp(new TrilinosSmoother(ifpackType, ifpackList));
      M.SetFactory("Smoother", rcp(new SmootherFactory(smootherPrototype)));
    }

    //
    // Setup preconditioner
    //

    int startLevel = 0;
    //      std::cout << startLevel << " " << optMaxLevels << std::endl;
    H->Setup(M, startLevel, optMaxLevels);

  } // end of Setup TimeMonitor

  /*{ // some debug output
    // print out content of levels
    std::cout << "FINAL CONTENT of multigrid levels" << std::endl;
    for(LO l = 0; l < H->GetNumLevels(); l++) {
      RCP<Level> coarseLevel = H->GetLevel(l);
      coarseLevel->print(*out);
    }
    std::cout << "END FINAL CONTENT of multigrid levels" << std::endl;
  } // end debug output*/

  //
  //
  // SOLVE
  //
  //

  // Define X, B
  RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
  RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);

  X->setSeed(846930886);
  X->randomize();
  A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
  B->norm2(norms);
  B->scale(1.0/norms[0]);

  //
  // Use AMG directly as an iterative method
  //

  if (optFixPoint) {

    X->putScalar( (SC) 0.0);

    TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 3 - Fixed Point Solve"));

    H->IsPreconditioner(false);
    H->Iterate(*B, *X, optIts);

  } // optFixedPt

  //
  // Use AMG as a preconditioner in Belos
  //

#ifdef HAVE_MUELU_BELOS

  if (optPrecond) {

    RCP<TimeMonitor> tm;
    tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
    // Operator and Multivector type that will be used with Belos
    typedef MultiVector          MV;
    typedef Belos::OperatorT<MV> OP;
    H->IsPreconditioner(true);

    // Define Operator and Preconditioner
    Teuchos::RCP<OP> belosOp   = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Operator object into a Belos operator
    Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H));  // Turns a MueLu::Hierarchy object into a Belos operator

    // Construct a Belos LinearProblem object
    RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
    belosProblem->setLeftPrec(belosPrec);

    bool set = belosProblem->setProblem();
    if (set == false) {
      if (comm->getRank() == 0)
        std::cout << std::endl << "ERROR:  Belos::LinearProblem failed to set up correctly!" << std::endl;
      return EXIT_FAILURE;
    }

    // Belos parameter list
    int maxIts = 100;
    Teuchos::ParameterList belosList;
    belosList.set("Maximum Iterations",    maxIts); // Maximum number of iterations allowed
    belosList.set("Convergence Tolerance", optTol);    // Relative convergence tolerance requested
    //belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
    belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
    belosList.set("Output Frequency", 1);
    belosList.set("Output Style", Belos::Brief);

    // Create an iterative solver manager
    RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));

    // Perform solve
    Belos::ReturnType ret = Belos::Unconverged;
    try {
      {
        TimeMonitor tm2(*TimeMonitor::getNewTimer("ScalingTest: 5bis - Belos Internal Solve"));
        ret = solver->solve();
      } // end of TimeMonitor

      // Get the number of iterations for this solve.
      if (comm->getRank() == 0)
        std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;

      // Compute actual residuals.
      int numrhs = 1;
      std::vector<double> actual_resids( numrhs ); //TODO: double?
      std::vector<double> rhs_norm( numrhs );
      RCP<MultiVector> resid = MultiVectorFactory::Build(map, numrhs);

      typedef Belos::OperatorTraits<SC, MV, OP>  OPT;
      typedef Belos::MultiVecTraits<SC, MV>     MVT;

      OPT::Apply( *belosOp, *X, *resid );
      MVT::MvAddMv( -1.0, *resid, 1.0, *B, *resid );
      MVT::MvNorm( *resid, actual_resids );
      MVT::MvNorm( *B, rhs_norm );
      *out<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
      for ( int i = 0; i<numrhs; i++) {
        double actRes = actual_resids[i]/rhs_norm[i];
        *out<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
        //if (actRes > tol) { badRes = true; }
      }

    } //try

    catch(...) {
      if (comm->getRank() == 0)
        std::cout << std::endl << "ERROR:  Belos threw an error! " << std::endl;
    }

    // Check convergence
    if (ret != Belos::Converged) {
      if (comm->getRank() == 0) std::cout << std::endl << "ERROR:  Belos did not converge! " << std::endl;
    } else {
      if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS:  Belos converged!" << std::endl;
    }
    tm = Teuchos::null;

  } //if (optPrecond)

#endif // HAVE_MUELU_BELOS

  //
  // Timer final summaries
  //

  globalTimeMonitor = Teuchos::null; // stop this timer before summary

  if (optTimings)
    TimeMonitor::summarize();

  //

  return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {

#ifdef HAVE_MPI
  MPI_Init(&argc, &argv);
  Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
  Epetra_SerialComm Comm;
#endif

  // initialize the random number generator

  int ml_one = 1;
  ML_srandom1(&ml_one);
  // ===================== //
  // create linear problem //
  // ===================== //

  ParameterList GaleriList;
  int base=10;
  GaleriList.set("nx", base);
  GaleriList.set("ny", base);
  GaleriList.set("nz", base * Comm.NumProc());
  GaleriList.set("mx", 1);
  GaleriList.set("my", 1);
  GaleriList.set("mz", Comm.NumProc());

  Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
  Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);

  Epetra_Vector LHS(*Map);
  Epetra_Vector RHS(*Map);
  
  Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);

  Teuchos::ParameterList MLList;
  double TotalErrorResidual = 0.0, TotalErrorExactSol = 0.0;

  // ==================n==== //
  // default options for SA //
  // ====================== //

  if (Comm.MyPID() == 0) PrintLine();

  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "Gauss-Seidel");
  char mystring[80];
  strcpy(mystring,"SA");
  TestMultiLevelPreconditioner(mystring, MLList, Problem, 
                               TotalErrorResidual, TotalErrorExactSol);

  // ============================== //
  // default options for SA, Jacobi //
  // ============================== //

  if (Comm.MyPID() == 0) PrintLine();

  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "Jacobi");

  TestMultiLevelPreconditioner(mystring, MLList, Problem, TotalErrorResidual,
                               TotalErrorExactSol);

  // =========================== //
  // default options for SA, Cheby //
  // =========================== //

  if (Comm.MyPID() == 0) PrintLine();

  ML_Epetra::SetDefaults("SA",MLList);
  MLList.set("smoother: type", "Chebyshev");

  TestMultiLevelPreconditioner(mystring, MLList, Problem, 
                               TotalErrorResidual, TotalErrorExactSol);
  // ===================== //
  // print out total error //
  // ===================== //

  
  if (Comm.MyPID() == 0) {
    cout << endl;
    cout << "......Total error for residual        = " << TotalErrorResidual << endl;
    cout << "......Total error for exact solution  = " << TotalErrorExactSol << endl;
    cout << endl;
  }

  
  delete Matrix;
  delete Map;
  
  if (TotalErrorResidual > 1e-8) {
    cerr << "Error: `MultiLevelPrecoditioner_Sym.exe' failed!" << endl;
    exit(EXIT_FAILURE);
  }

#ifdef HAVE_MPI
  MPI_Finalize();
#endif

  if (Comm.MyPID() == 0)
    cerr << "`MultiLevelPrecoditioner_Sym.exe' passed!" << endl;

  return (EXIT_SUCCESS);
}
Exemple #15
0
int main(int argc, char *argv[]) {
  //
  Teuchos::GlobalMPISession session(&argc, &argv, NULL);
  //
  typedef double                            ST;
  typedef Teuchos::ScalarTraits<ST>        SCT;
  typedef SCT::magnitudeType                MT;
  typedef Epetra_MultiVector                MV;
  typedef Epetra_Operator                   OP;
  typedef Belos::MultiVecTraits<ST,MV>     MVT;
  typedef Belos::OperatorTraits<ST,MV,OP>  OPT;

  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;

  bool verbose = false;
  bool success = true;
  try {
    bool proc_verbose = false;
    bool pseudo = false;   // use pseudo block GMRES to solve this linear system.
    int frequency = -1;
    int blocksize = 1;
    int numrhs = 1;
    int maxrestarts = 15; // number of restarts allowed
    int maxiters = -1;    // maximum number of iterations allowed per linear system
    std::string filename("orsirr1.hb");
    MT tol = 1.0e-5;  // relative residual tolerance

    Teuchos::CommandLineProcessor cmdp(false,true);
    cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
    cmdp.setOption("pseudo","regular",&pseudo,"Use pseudo-block GMRES to solve the linear systems.");
    cmdp.setOption("frequency",&frequency,"Solvers frequency for printing residuals (#iters).");
    cmdp.setOption("filename",&filename,"Filename for Harwell-Boeing test matrix.");
    cmdp.setOption("tol",&tol,"Relative residual tolerance used by GMRES solver.");
    cmdp.setOption("max-restarts",&maxrestarts,"Maximum number of restarts allowed for GMRES solver.");
    cmdp.setOption("blocksize",&blocksize,"Block size used by GMRES.");
    cmdp.setOption("maxiters",&maxiters,"Maximum number of iterations per linear system (-1 = adapted to problem/block size).");
    if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
      return -1;
    }
    if (!verbose)
      frequency = -1;  // reset frequency if test is not verbose
    //
    // Get the problem
    //
    int MyPID;
    RCP<Epetra_CrsMatrix> A;
    RCP<Epetra_MultiVector> B, X;
    int return_val =Belos::createEpetraProblem(filename,NULL,&A,&B,&X,&MyPID);
    if(return_val != 0) return return_val;
    const Epetra_Map &Map = A->RowMap();
    proc_verbose = verbose && (MyPID==0);  /* Only print on the zero processor */
    //
    // ********Other information used by block solver***********
    // *****************(can be user specified)******************
    //
    const int NumGlobalElements = B->GlobalLength();
    if (maxiters == -1)
      maxiters = NumGlobalElements/blocksize - 1; // maximum number of iterations to run
    //
    ParameterList belosList;
    belosList.set( "Num Blocks", maxiters );               // Maximum number of blocks in Krylov factorization
    belosList.set( "Block Size", blocksize );              // Blocksize to be used by iterative solver
    belosList.set( "Maximum Iterations", maxiters );       // Maximum number of iterations allowed
    belosList.set( "Maximum Restarts", maxrestarts );      // Maximum number of restarts allowed
    belosList.set( "Convergence Tolerance", tol );         // Relative convergence tolerance requested
    if (verbose) {
      belosList.set( "Verbosity", Belos::Errors + Belos::Warnings +
          Belos::TimingDetails + + Belos::StatusTestDetails );
      if (frequency > 0)
        belosList.set( "Output Frequency", frequency );
    }
    else
      belosList.set( "Verbosity", Belos::Errors + Belos::Warnings );
    //
    //
    // Construct an unpreconditioned linear problem instance.
    //
    Belos::LinearProblem<double,MV,OP> problem( A, X, B );
    bool set = problem.setProblem();
    if (set == false) {
      if (proc_verbose)
        std::cout << std::endl << "ERROR:  Belos::LinearProblem failed to set up correctly!" << std::endl;
      return -1;
    }
    //
    // *******************************************************************
    // *************Start the block Gmres iteration*************************
    // *******************************************************************
    //
    Teuchos::RCP< Belos::SolverManager<double,MV,OP> > solver;
    if (pseudo)
      solver = Teuchos::rcp( new Belos::PseudoBlockGmresSolMgr<double,MV,OP>( rcp(&problem,false), rcp(&belosList,false) ) );
    else
      solver = Teuchos::rcp( new Belos::BlockGmresSolMgr<double,MV,OP>( rcp(&problem,false), rcp(&belosList,false) ) );
    //
    // **********Print out information about problem*******************
    //
    if (proc_verbose) {
      std::cout << std::endl << std::endl;
      std::cout << "Dimension of matrix: " << NumGlobalElements << std::endl;
      std::cout << "Number of right-hand sides: " << numrhs << std::endl;
      std::cout << "Block size used by solver: " << blocksize << std::endl;
      std::cout << "Number of restarts allowed: " << maxrestarts << std::endl;
      std::cout << "Max number of Gmres iterations per restart cycle: " << maxiters << std::endl;
      std::cout << "Relative residual tolerance: " << tol << std::endl;
      std::cout << std::endl;
    }
    //
    // Perform solve
    //
    Belos::ReturnType ret = solver->solve();

    //
    // Compute actual residuals.
    //
    std::vector<double> actual_resids( numrhs );
    std::vector<double> rhs_norm( numrhs );
    Epetra_MultiVector resid(Map, numrhs);
    OPT::Apply( *A, *X, resid );
    MVT::MvAddMv( -1.0, resid, 1.0, *B, resid );
    MVT::MvNorm( resid, actual_resids );
    MVT::MvNorm( *B, rhs_norm );
    if (proc_verbose) {
      std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
      for ( int i=0; i<numrhs; i++) {
        std::cout<<"Problem "<<i<<" : \t"<< actual_resids[i]/rhs_norm[i] <<std::endl;
      }
    }

    success = ret==Belos::Converged;

    if (success) {
      if (proc_verbose)
        std::cout << "End Result: TEST PASSED" << std::endl;
    } else {
      if (proc_verbose)
        std::cout << "End Result: TEST FAILED" << std::endl;
    }
  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);

  return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
} // end test_bl_gmres_hb.cpp
int
main (int argc, char *argv[])
{
  using Teuchos::Comm;
  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::rcpFromRef;
  using std::endl;
  using std::cout;

  typedef double                           ST;
  typedef Teuchos::ScalarTraits<ST>       STS;
  typedef STS::magnitudeType               MT;
  typedef Tpetra::Operator<ST,int>         OP;
  typedef Tpetra::MultiVector<ST,int>      MV;
  typedef Belos::OperatorTraits<ST,MV,OP> OPT;
  typedef Belos::MultiVecTraits<ST,MV>    MVT;
  typedef MV::node_type             node_type;

  typedef Teuchos::CommandLineProcessor   CLP;



  Teuchos::GlobalMPISession mpisess (&argc, &argv, &cout);

  bool success = false;
  bool verbose = false;
  try {
    RCP<const Comm<int> > comm =
      Tpetra::DefaultPlatform::getDefaultPlatform ().getComm ();
    RCP<node_type> node =
      Tpetra::DefaultPlatform::getDefaultPlatform ().getNode ();

    //
    // Get test parameters from command-line processor
    //
    bool proc_verbose = false;
    bool debug = false;
    int frequency = -1;
    int numrhs = 1;
    int blocksize = 1;
    int maxiters = -1;
    std::string filename ("bcsstk14.hb");
    MT tol = 1.0e-5;

    CLP cmdp (false, true);
    cmdp.setOption ("verbose", "quiet", &verbose, "Print messages and "
                    "results.");
    cmdp.setOption ("debug", "nodebug", &debug, "Run debugging checks.");
    cmdp.setOption ("frequency", &frequency, "Solver's frequency for printing "
                    "residuals (#iters).  -1 means \"never\"; 1 means \"every "
                    "iteration.\"");
    cmdp.setOption ("tol", &tol, "Relative residual tolerance used by solver.");
    cmdp.setOption ("filename", &filename, "Filename for Harwell-Boeing test "
                    "matrix.");
    cmdp.setOption ("num-rhs", &numrhs, "Number of right-hand sides for which "
                    "to solve.");
    cmdp.setOption ("max-iters", &maxiters, "Maximum number of iterations per "
                    "linear system (-1 := adapted to problem/block size).");
    cmdp.setOption ("block-size", &blocksize, "Block size to be used by the "
                    "solver.");
    if (cmdp.parse (argc, argv) != CLP::PARSE_SUCCESSFUL) {
      return -1;
    }
    if (debug) {
      verbose = true;
    }
    if (!verbose) {
      frequency = -1;  // reset frequency if test is not verbose
    }

    const int MyPID = comm->getRank ();
    proc_verbose = verbose && MyPID == 0;

    if (proc_verbose) {
      cout << Belos::Belos_Version () << endl << endl;
    }

    //
    // Get the data from the HB file and build the Map,Matrix
    //
    RCP<Tpetra::CrsMatrix<ST,int> > A;
    Tpetra::Utils::readHBMatrix (filename, comm, node, A);
    RCP<const Tpetra::Map<int> > map = A->getDomainMap ();

    // Create initial vectors
    RCP<MV> B, X;
    X = rcp (new MV (map, numrhs));
    MVT::MvRandom (*X);
    B = rcp (new MV (map, numrhs));
    OPT::Apply (*A, *X, *B);
    MVT::MvInit (*X, STS::zero ());

    //
    // ********Other information used by block solver***********
    // *****************(can be user specified)******************
    //
    const int NumGlobalElements = B->getGlobalLength();
    if (maxiters == -1) {
      maxiters = NumGlobalElements/blocksize - 1; // maximum number of iterations to run
    }
    //
    ParameterList belosList;
    belosList.set( "Block Size", blocksize );              // Blocksize to be used by iterative solver
    belosList.set( "Maximum Iterations", maxiters );       // Maximum number of iterations allowed
    belosList.set( "Convergence Tolerance", tol );         // Relative convergence tolerance requested
    int verbLevel = Belos::Errors + Belos::Warnings;
    if (debug) {
      verbLevel += Belos::Debug;
    }
    if (verbose) {
      verbLevel += Belos::TimingDetails + Belos::FinalSummary + Belos::StatusTestDetails;
    }
    belosList.set( "Verbosity", verbLevel );
    if (verbose) {
      if (frequency > 0) {
        belosList.set( "Output Frequency", frequency );
      }
    }
    //
    // Construct an unpreconditioned linear problem instance.
    //
    Belos::LinearProblem<ST,MV,OP> problem( A, X, B );
    bool set = problem.setProblem ();
    if (! set) {
      if (proc_verbose) {
        cout << endl << "ERROR:  Belos::LinearProblem failed to set up correctly!" << endl;
      }
      return EXIT_FAILURE;
    }
    //
    // *******************************************************************
    // *************Start the block CG iteration***********************
    // *******************************************************************
    //
    typedef Belos::PseudoBlockStochasticCGSolMgr<ST,MV,OP> solver_type;
    solver_type solver (rcpFromRef (problem), rcpFromRef (belosList));

    //
    // **********Print out information about problem*******************
    //
    if (proc_verbose) {
      cout << endl << endl;
      cout << "Dimension of matrix: " << NumGlobalElements << endl;
      cout << "Number of right-hand sides: " << numrhs << endl;
      cout << "Block size used by solver: " << blocksize << endl;
      cout << "Max number of CG iterations: " << maxiters << endl;
      cout << "Relative residual tolerance: " << tol << endl;
      cout << endl;
    }
    //
    // Perform solve
    //
    Belos::ReturnType ret = solver.solve();
    //
    // Compute actual residuals.
    //
    bool badRes = false;
    std::vector<MT> actual_resids (numrhs);
    std::vector<MT> rhs_norm (numrhs);
    MV resid (map, numrhs);
    OPT::Apply (*A, *X, resid);
    MVT::MvAddMv (-STS::one (), resid, STS::one (), *B, resid);
    MVT::MvNorm (resid, actual_resids);
    MVT::MvNorm (*B, rhs_norm);
    if (proc_verbose) {
      cout << "---------- Actual Residuals (normalized) ----------" << endl << endl;
    }
    for (int i=0; i<numrhs; i++) {
      MT actRes = actual_resids[i]/rhs_norm[i];
      if (proc_verbose) {
        cout << "Problem " << i << " : \t" << actRes << endl;
      }
      if (actRes > tol) badRes = true;
    }

    success = ret == Belos::Converged && ! badRes;
    if (success) {
      if (proc_verbose) {
        cout << "\nEnd Result: TEST PASSED" << endl;
      }
    } else {
      if (proc_verbose) {
        cout << "\nEnd Result: TEST FAILED" << endl;
      }
    }
  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);

  return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
int main(int argc, char *argv[]) {
  //
  int MyPID = 0;
#ifdef EPETRA_MPI
  // Initialize MPI
  MPI_Init(&argc,&argv);
  Epetra_MpiComm Comm(MPI_COMM_WORLD);
  MyPID = Comm.MyPID();
#else
  Epetra_SerialComm Comm;
#endif
  //
  typedef double                            ST;
  typedef Teuchos::ScalarTraits<ST>        SCT;
  typedef SCT::magnitudeType                MT;
  typedef Epetra_MultiVector                MV;
  typedef Epetra_Operator                   OP;
  typedef Belos::MultiVecTraits<ST,MV>     MVT;
  typedef Belos::OperatorTraits<ST,MV,OP>  OPT;

  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;

bool verbose = false;
bool success = true;
try {
bool proc_verbose = false;
  bool debug = false;
  bool leftprec = true;      // left preconditioning or right.
  int frequency = -1;        // frequency of status test output.
  int numrhs = 1;            // number of right-hand sides to solve for
  int maxiters = -1;         // maximum number of iterations allowed per linear system
  int maxsubspace = 250;     // maximum number of blocks the solver can use for the subspace
  int recycle = 50;          // maximum size of recycle space
  int maxrestarts = 15;      // maximum number of restarts allowed
  std::string filename("sherman5.hb");
  std::string ortho("IMGS");
  MT tol = 1.0e-10;          // relative residual tolerance

  Teuchos::CommandLineProcessor cmdp(false,true);
  cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
  cmdp.setOption("debug","nondebug",&debug, "Print debugging information from solver.");
  cmdp.setOption("left-prec","right-prec",&leftprec,"Left preconditioning or right.");
  cmdp.setOption("frequency",&frequency,"Solvers frequency for printing residuals (#iters).");
  cmdp.setOption("filename",&filename,"Filename for test matrix.  Acceptable file extensions: *.hb,*.mtx,*.triU,*.triS");
  cmdp.setOption("tol",&tol,"Relative residual tolerance used by GMRES solver.");
  cmdp.setOption("num-rhs",&numrhs,"Number of right-hand sides to be solved for.");
  cmdp.setOption("max-iters",&maxiters,"Maximum number of iterations per linear system (-1 = adapted to problem/block size).");
  cmdp.setOption("max-subspace",&maxsubspace,"Maximum number of blocks the solver can use for the subspace.");
  cmdp.setOption("recycle",&recycle,"Number of vectors in recycle space.");
  cmdp.setOption("max-cycles",&maxrestarts,"Maximum number of cycles allowed for GCRO-DR solver.");
  cmdp.setOption("ortho-type",&ortho,"Orthogonalization type. Must be one of DGKS, ICGS, IMGS.");
  if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
    return -1;
  }
  if (!verbose)
    frequency = -1;  // reset frequency if test is not verbose

  //
  // *************Get the problem*********************
  //
  RCP<Epetra_CrsMatrix> A;
  RCP<Epetra_MultiVector> B, X;
  int return_val =Belos::Util::createEpetraProblem(filename,NULL,&A,NULL,NULL,&MyPID);
  const Epetra_Map &Map = A->RowMap();
  if(return_val != 0) return return_val;
  proc_verbose = verbose && (MyPID==0); /* Only print on zero processor */
  X = rcp( new Epetra_MultiVector( Map, numrhs ) );
  B = rcp( new Epetra_MultiVector( Map, numrhs ) );
  X->Random();
  OPT::Apply( *A, *X, *B );
  X->PutScalar( 0.0 );
  //
  // ************Construct preconditioner*************
  //
  if (proc_verbose) std::cout << std::endl << std::endl;
  if (proc_verbose) std::cout << "Constructing ILU preconditioner" << std::endl;
  int Lfill = 2;
  if (proc_verbose) std::cout << "Using Lfill = " << Lfill << std::endl;
  int Overlap = 2;
  if (proc_verbose) std::cout << "Using Level Overlap = " << Overlap << std::endl;
  double Athresh = 0.0;
  if (proc_verbose) std::cout << "Using Absolute Threshold Value of " << Athresh << std::endl;
  double Rthresh = 1.0;
  if (proc_verbose) std::cout << "Using Relative Threshold Value of " << Rthresh << std::endl;
  //
  Teuchos::RCP<Ifpack_IlukGraph> ilukGraph;
  Teuchos::RCP<Ifpack_CrsRiluk> ilukFactors;
  //
  ilukGraph = Teuchos::rcp(new Ifpack_IlukGraph(A->Graph(), Lfill, Overlap));
  int info = ilukGraph->ConstructFilledGraph();
  assert( info == 0 );
  ilukFactors = Teuchos::rcp(new Ifpack_CrsRiluk(*ilukGraph));
  int initerr = ilukFactors->InitValues(*A);
  if (initerr != 0) std::cout << "InitValues error = " << initerr;
  info = ilukFactors->Factor();
  assert( info == 0 );
  bool transA = false;
  double Cond_Est;
  ilukFactors->Condest(transA, Cond_Est);
  if (proc_verbose) {
    std::cout << "Condition number estimate for this preconditoner = " << Cond_Est << std::endl;
    std::cout << std::endl;
  }

  // Create the Belos preconditioned operator from the Ifpack preconditioner.
  // NOTE:  This is necessary because Belos expects an operator to apply the
  //        preconditioner with Apply() NOT ApplyInverse().
  RCP<Belos::EpetraPrecOp> belosPrec = rcp( new Belos::EpetraPrecOp( ilukFactors ) );

  //
  // ********Other information used by block solver***********
  // *****************(can be user specified)******************
  //
  const int NumGlobalElements = B->GlobalLength();
  if (maxiters == -1)
    maxiters = NumGlobalElements - 1; // maximum number of iterations to run
  //
  ParameterList belosList;
  belosList.set( "Num Blocks", maxsubspace );            // Maximum number of blocks in Krylov factorization
  belosList.set( "Maximum Iterations", maxiters );       // Maximum number of iterations allowed
  belosList.set( "Maximum Restarts", maxrestarts );      // Maximum number of restarts allowed
  belosList.set( "Convergence Tolerance", tol );         // Relative convergence tolerance requested
  belosList.set( "Num Recycled Blocks", recycle );       // Number of vectors in recycle space
  belosList.set( "Orthogonalization", ortho );           // Orthogonalization type
  if (numrhs > 1) {
    belosList.set( "Show Maximum Residual Norm Only", true );  // Show only the maximum residual norm
  }
  int verbosity = Belos::Errors + Belos::Warnings;
  if (verbose) {
    verbosity += Belos::TimingDetails + Belos::StatusTestDetails;
    if (frequency > 0)
      belosList.set( "Output Frequency", frequency );
  }
  if (debug) {
    verbosity += Belos::Debug;
  }
  belosList.set( "Verbosity", verbosity );
  //
  // *******Construct a preconditioned linear problem********
  //
  RCP<Belos::LinearProblem<double,MV,OP> > problem
    = rcp( new Belos::LinearProblem<double,MV,OP>( A, X, B ) );
  if (leftprec) {
    problem->setLeftPrec( belosPrec );
  }
  else {
    problem->setRightPrec( belosPrec );
  }
  bool set = problem->setProblem();
  if (set == false) {
    if (proc_verbose)
      std::cout << std::endl << "ERROR:  Belos::LinearProblem failed to set up correctly!" << std::endl;
    return -1;
  }

  // Create an iterative solver manager.
  RCP< Belos::SolverManager<double,MV,OP> > solver
    = rcp( new Belos::GCRODRSolMgr<double,MV,OP>(problem, rcp(&belosList,false)));

  //
  // *******************************************************************
  // *************Start the block Gmres iteration*************************
  // *******************************************************************
  //
  if (proc_verbose) {
    std::cout << std::endl << std::endl;
    std::cout << "Dimension of matrix: " << NumGlobalElements << std::endl;
    std::cout << "Number of right-hand sides: " << numrhs << std::endl;
    std::cout << "Number of restarts allowed: " << maxrestarts << std::endl;
    std::cout << "Max number of Gmres iterations per restart cycle: " << maxiters << std::endl;
    std::cout << "Relative residual tolerance: " << tol << std::endl;
    std::cout << std::endl;
  }
  //
  // Perform solve
  //
  Belos::ReturnType ret = solver->solve();
  //
  // Compute actual residuals.
  //
  bool badRes = false;
  std::vector<double> actual_resids( numrhs );
  std::vector<double> rhs_norm( numrhs );
  Epetra_MultiVector resid(Map, numrhs);
  OPT::Apply( *A, *X, resid );
  MVT::MvAddMv( -1.0, resid, 1.0, *B, resid );
  MVT::MvNorm( resid, actual_resids );
  MVT::MvNorm( *B, rhs_norm );
  if (proc_verbose) {
    std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
    for ( int i=0; i<numrhs; i++) {
      double actRes = actual_resids[i]/rhs_norm[i];
      std::cout<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
      if (actRes > tol) badRes = true;
    }
  }

if (ret!=Belos::Converged || badRes) {
  success = false;
  if (proc_verbose)
    std::cout << std::endl << "ERROR:  Belos did not converge!" << std::endl;
} else {
  success = true;
  if (proc_verbose)
    std::cout << std::endl << "SUCCESS:  Belos converged!" << std::endl;
}
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);

#ifdef EPETRA_MPI
MPI_Finalize();
#endif

return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
int main(int argc, char *argv[]) {
  //
  int MyPID = 0;
#ifdef EPETRA_MPI
  // Initialize MPI
  MPI_Init(&argc,&argv);
  Epetra_MpiComm Comm(MPI_COMM_WORLD);
  MyPID = Comm.MyPID();
#else
  Epetra_SerialComm Comm;
#endif
  //
  typedef double                            ST;
  typedef Teuchos::ScalarTraits<ST>        SCT;
  typedef SCT::magnitudeType                MT;
  typedef Epetra_MultiVector                MV;
  typedef Epetra_Operator                   OP;
  typedef Belos::MultiVecTraits<ST,MV>     MVT;
  typedef Belos::OperatorTraits<ST,MV,OP>  OPT;

  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;

  bool verbose = false, debug = false, proc_verbose = false;
  int frequency = -1;        // frequency of status test output.
  int blocksize = 1;         // blocksize
  int numrhs = 1;            // number of right-hand sides to solve for
  int maxiters = -1;         // maximum number of iterations allowed per linear system
  int maxsubspace = 50;      // maximum number of blocks the solver can use for the subspace
  int maxrestarts = 15;      // number of restarts allowed 
  std::string filename("orsirr1.hb");
  MT tol = 1.0e-5;           // relative residual tolerance

  Teuchos::CommandLineProcessor cmdp(false,true);
  cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
  cmdp.setOption("debug","nondebug",&debug,"Print debugging information from solver.");
  cmdp.setOption("frequency",&frequency,"Solvers frequency for printing residuals (#iters).");
  cmdp.setOption("filename",&filename,"Filename for test matrix.  Acceptable file extensions: *.hb,*.mtx,*.triU,*.triS");
  cmdp.setOption("tol",&tol,"Relative residual tolerance used by GMRES solver.");
  cmdp.setOption("num-rhs",&numrhs,"Number of right-hand sides to be solved for.");
  cmdp.setOption("block-size",&blocksize,"Block size used by GMRES.");
  cmdp.setOption("max-iters",&maxiters,"Maximum number of iterations per linear system (-1 = adapted to problem/block size).");
  cmdp.setOption("max-subspace",&maxsubspace,"Maximum number of blocks the solver can use for the subspace.");
  cmdp.setOption("max-restarts",&maxrestarts,"Maximum number of restarts allowed for GMRES solver.");
  if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
    return -1;
  }
  if (!verbose)
    frequency = -1;  // reset frequency if test is not verbose
  //
  // Get the problem
  //
  RCP<Epetra_Map> Map;
  RCP<Epetra_CrsMatrix> A;
  RCP<Epetra_MultiVector> B, X;
  RCP<Epetra_Vector> vecB, vecX;
  EpetraExt::readEpetraLinearSystem(filename, Comm, &A, &Map, &vecX, &vecB);
  A->OptimizeStorage();
  proc_verbose = verbose && (MyPID==0);  /* Only print on the zero processor */

  // Check to see if the number of right-hand sides is the same as requested.
  if (numrhs>1) {
    X = rcp( new Epetra_MultiVector( *Map, numrhs ) );
    B = rcp( new Epetra_MultiVector( *Map, numrhs ) );
    X->Seed();
    X->Random();
    OPT::Apply( *A, *X, *B );
    X->PutScalar( 0.0 );
  }
  else {
    X = Teuchos::rcp_implicit_cast<Epetra_MultiVector>(vecX);
    B = Teuchos::rcp_implicit_cast<Epetra_MultiVector>(vecB);
  }
  //
  // ********Other information used by block solver***********
  // *****************(can be user specified)******************
  //
  const int NumGlobalElements = B->GlobalLength();
  if (maxiters == -1)
    maxiters = NumGlobalElements/blocksize - 1; // maximum number of iterations to run
  //
  ParameterList belosList;
  belosList.set( "Num Blocks", maxsubspace);             // Maximum number of blocks in Krylov factorization
  belosList.set( "Block Size", blocksize );              // Blocksize to be used by iterative solver
  belosList.set( "Maximum Iterations", maxiters );       // Maximum number of iterations allowed
  belosList.set( "Maximum Restarts", maxrestarts );      // Maximum number of restarts allowed
  belosList.set( "Convergence Tolerance", tol );         // Relative convergence tolerance requested
  int verbosity = Belos::Errors + Belos::Warnings;
  if (verbose) {
    verbosity += Belos::TimingDetails + Belos::StatusTestDetails;
    if (frequency > 0)
      belosList.set( "Output Frequency", frequency );
  }
  if (debug) {
    verbosity += Belos::Debug;
  }
  belosList.set( "Verbosity", verbosity );
  //
  // Construct an unpreconditioned linear problem instance.
  //
  Belos::LinearProblem<double,MV,OP> problem( A, X, B );
  bool set = problem.setProblem();
  if (set == false) {
    if (proc_verbose)
      std::cout << std::endl << "ERROR:  Belos::LinearProblem failed to set up correctly!" << std::endl;
    return -1;
  }
  //
  // *******************************************************************
  // *************Start the block Gmres iteration*************************
  // *******************************************************************
  //
  // Create an iterative solver manager.
  RCP< Belos::SolverManager<double,MV,OP> > newSolver
    = rcp( new Belos::BlockGmresSolMgr<double,MV,OP>(rcp(&problem,false), rcp(&belosList,false)));

  //
  // **********Print out information about problem*******************
  //
  if (proc_verbose) {
    std::cout << std::endl << std::endl;
    std::cout << "Dimension of matrix: " << NumGlobalElements << std::endl;
    std::cout << "Number of right-hand sides: " << numrhs << std::endl;
    std::cout << "Block size used by solver: " << blocksize << std::endl;
    std::cout << "Max number of restarts allowed: " << maxrestarts << std::endl;
    std::cout << "Max number of Gmres iterations per linear system: " << maxiters << std::endl; 
    std::cout << "Relative residual tolerance: " << tol << std::endl;
    std::cout << std::endl;
  }
  //
  // Perform solve
  //
  Belos::ReturnType ret = newSolver->solve();
  //
  // Get the number of iterations for this solve.
  //
  int numIters = newSolver->getNumIters();
  if (proc_verbose)
    std::cout << "Number of iterations performed for this solve: " << numIters << std::endl;
  //
  // Compute actual residuals.
  //
  bool badRes = false;
  std::vector<double> actual_resids( numrhs );
  std::vector<double> rhs_norm( numrhs );
  Epetra_MultiVector resid(*Map, numrhs);
  OPT::Apply( *A, *X, resid );
  MVT::MvAddMv( -1.0, resid, 1.0, *B, resid ); 
  MVT::MvNorm( resid, actual_resids );
  MVT::MvNorm( *B, rhs_norm );
  if (proc_verbose) {
    std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
    for ( int i=0; i<numrhs; i++) {
      double actRes = actual_resids[i]/rhs_norm[i];
      std::cout<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
      if (actRes > tol) badRes = true;
    }
  }

#ifdef EPETRA_MPI
  MPI_Finalize();
#endif

  if (ret!=Belos::Converged || badRes) {
    if (proc_verbose)
      std::cout << std::endl << "ERROR:  Belos did not converge!" << std::endl;	
    return -1;
  }
  //
  // Default return value
  //
  if (proc_verbose)
    std::cout << std::endl << "SUCCESS:  Belos converged!" << std::endl;
  return 0;
  
  //
} 
Exemple #19
0
Teuchos::ParameterList * ML_Epetra::GetValidRefMaxwellParameters(){
  using Teuchos::ParameterList;
  using Teuchos::setStringToIntegralParameter;
  using Teuchos::tuple;
  using std::string;

  ParameterList dummy,List11;
  ParameterList * PL = GetValidMLPParameters();

  /* RefMaxwell Options */
  setStringToIntegralParameter<int>("refmaxwell: 11solver","edge matrix free","(1,1) Block Solver",tuple<string>("edge matrix free"),PL);
  setStringToIntegralParameter<int>("refmaxwell: 22solver","multilevel","(2,2) Block Solver",tuple<string>("multilevel"),PL);
  setStringToIntegralParameter<int>("refmaxwell: mode","additive","Mode for RefMaxwell",tuple<string>("additive","212","121"),PL);
  PL->set("edge matrix free: coarse",dummy);
  List11.set("aggregation: aux: user matrix",(Epetra_CrsMatrix*)0);
  PL->set("refmaxwell: 11list",List11);
  PL->set("refmaxwell: 22list",dummy);


  // HAQ - clobber the smoother list to avoid validation.  MUST FIX LATER
  PL->set("smoother: type","IFPACK");


  /* RefMaxwell Options - Unsupported */
  PL->set("refmaxwell: aggregate with sigma",false);
  PL->set("refmaxwell: lump m1",false);
  PL->set("refmaxwell: disable addon",false); 
  PL->set("refmaxwell: normalize prolongator",false);
  PL->set("refmaxwell: parallelize blocks",false);
  PL->set("refmaxwell: local nodal list",dummy);
  PL->set("refmaxwell: enable local nodal solver",false);
  PL->set("refmaxwell: global to local nodal transfer matrix",(Epetra_CrsMatrix*)0);  
  PL->set("refmaxwell: drop nodal correction",false);//HAQ
  return PL;
}
void panzer::EquationSet_DefaultImpl<EvalT>::
buildAndRegisterDOFProjectionsToIPEvaluators(PHX::FieldManager<panzer::Traits>& fm,
                                             const panzer::FieldLayoutLibrary& fl,
                                             const Teuchos::RCP<panzer::IntegrationRule>& ir,
                                             const Teuchos::Ptr<const panzer::LinearObjFactory<panzer::Traits> > & lof,
                                             const Teuchos::ParameterList& user_data) const
{
  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;

  Teuchos::RCP<const panzer::UniqueGlobalIndexerBase> globalIndexer;
  if(lof!=Teuchos::null) 
    globalIndexer = lof->getUniqueGlobalIndexerBase();
  
  // DOFs: Scalar value @ basis --> Scalar value @ IP 
  for (DescriptorIterator dof_iter = m_provided_dofs_desc.begin(); dof_iter != m_provided_dofs_desc.end(); ++dof_iter) {

    ParameterList p;
    p.set("Name", dof_iter->first);
    p.set("Basis", fl.lookupLayout(dof_iter->first));
    p.set("IR", ir);

    if(globalIndexer!=Teuchos::null) {
      // build the offsets for this field
      int fieldNum = globalIndexer->getFieldNum(dof_iter->first);
      RCP<const std::vector<int> > offsets = 
          rcp(new std::vector<int>(globalIndexer->getGIDFieldOffsets(m_block_id,fieldNum)));
      p.set("Jacobian Offsets Vector", offsets);
    }
    // else default to the slow DOF call
    
    RCP< PHX::Evaluator<panzer::Traits> > op = 
      rcp(new panzer::DOF<EvalT,panzer::Traits>(p));
    
    this->template registerEvaluator<EvalT>(fm, op);
  }

  // Gradients of DOFs: Scalar value @ basis --> Vector value @ IP

  for(typename std::map<std::string,DOFDescriptor>::const_iterator itr=m_provided_dofs_desc.begin();
      itr!=m_provided_dofs_desc.end();++itr) {
    
    if(itr->second.basis->supportsGrad()) {
      
      // is gradient required for this variable
      if(!itr->second.grad.first) 
        continue; // its not required, quit the loop

      const std::string dof_name =      itr->first;
      const std::string dof_grad_name = itr->second.grad.second;

      ParameterList p;
      p.set("Name", dof_name);
      p.set("Gradient Name", dof_grad_name);
      p.set("Basis", fl.lookupLayout(dof_name)); 
      p.set("IR", ir);
      
      RCP< PHX::Evaluator<panzer::Traits> > op = 
        rcp(new panzer::DOFGradient<EvalT,panzer::Traits>(p));

      this->template registerEvaluator<EvalT>(fm, op);
    }
  }

  // Curl of DOFs: Vector value @ basis --> Vector value @ IP (3D) or Scalar value @ IP (2D)

  for(typename std::map<std::string,DOFDescriptor>::const_iterator itr=m_provided_dofs_desc.begin();
      itr!=m_provided_dofs_desc.end();++itr) {
    
    if(itr->second.basis->supportsCurl()) {

      // is curl required for this variable
      if(!itr->second.curl.first) 
        continue; // its not required, quit the loop

      const std::string dof_name =      itr->first;
      const std::string dof_curl_name = itr->second.curl.second;

      ParameterList p;
      p.set("Name", dof_name);
      p.set("Curl Name", dof_curl_name);
      p.set("Basis", fl.lookupLayout(dof_name)); 
      p.set("IR", ir);

      // this will help accelerate the DOFCurl evaluator when Jacobians are needed
      if(globalIndexer!=Teuchos::null) {
        // build the offsets for this field
        int fieldNum = globalIndexer->getFieldNum(dof_name);
        RCP<const std::vector<int> > offsets = 
            rcp(new std::vector<int>(globalIndexer->getGIDFieldOffsets(m_block_id,fieldNum)));
        p.set("Jacobian Offsets Vector", offsets);
      }
      // else default to the slow DOF call
    
      
      RCP< PHX::Evaluator<panzer::Traits> > op = 
        rcp(new panzer::DOFCurl<EvalT,panzer::Traits>(p));

      this->template registerEvaluator<EvalT>(fm, op);
    }

  }

  // Div of DOFs: Vector value @ basis --> Scalar value @ IP

  for(typename std::map<std::string,DOFDescriptor>::const_iterator itr=m_provided_dofs_desc.begin();
      itr!=m_provided_dofs_desc.end();++itr) {

    if(itr->second.basis->supportsDiv()) {

      // is div required for this variable
      if(!itr->second.div.first) 
        continue; // its not required, quit the loop

      const std::string dof_name =      itr->first;
      const std::string dof_div_name = itr->second.div.second;

      ParameterList p;
      p.set("Name", dof_name);
      p.set("Div Name", dof_div_name);
      p.set("Basis", fl.lookupLayout(dof_name)); 
      p.set("IR", ir);

      // this will help accelerate the DOFDiv evaluator when Jacobians are needed
      if(globalIndexer!=Teuchos::null) {
        // build the offsets for this field
        int fieldNum = globalIndexer->getFieldNum(dof_name);
        RCP<const std::vector<int> > offsets = 
            rcp(new std::vector<int>(globalIndexer->getGIDFieldOffsets(m_block_id,fieldNum)));
        p.set("Jacobian Offsets Vector", offsets);
      }
      // else default to the slow DOF call
    
      
      RCP< PHX::Evaluator<panzer::Traits> > op = 
        rcp(new panzer::DOFDiv<EvalT,panzer::Traits>(p));

      this->template registerEvaluator<EvalT>(fm, op);
    }
  }

  // Time derivative of DOFs: Scalar value @ basis --> Scalar value @ IP 
  for(typename std::map<std::string,DOFDescriptor>::const_iterator itr=m_provided_dofs_desc.begin();
      itr!=m_provided_dofs_desc.end();++itr) {
    // is td required for this variable
    if(!itr->second.timeDerivative.first) 
      continue; // its not required, quit the loop

    const std::string td_name = itr->second.timeDerivative.second;

    ParameterList p;
    p.set("Name", td_name);
    p.set("Basis", fl.lookupLayout(itr->first)); 
    p.set("IR", ir);

    if(globalIndexer!=Teuchos::null) {
      // build the offsets for this field
      int fieldNum = globalIndexer->getFieldNum(itr->first);
      RCP<const std::vector<int> > offsets = 
          rcp(new std::vector<int>(globalIndexer->getGIDFieldOffsets(m_block_id,fieldNum)));
      p.set("Jacobian Offsets Vector", offsets);
    }
    // else default to the slow DOF call

    // set the orientiation field name explicitly if orientations are
    // required for the basis
    if(itr->second.basis->requiresOrientations())
      p.set("Orientation Field Name", itr->first+" Orientation");
    
    RCP< PHX::Evaluator<panzer::Traits> > op = 
      rcp(new panzer::DOF<EvalT,panzer::Traits>(p));
    
    this->template registerEvaluator<EvalT>(fm, op);
  }

}
Exemple #21
0
Teuchos::ParameterList * ML_Epetra::GetValidMLPParameters(){
  using Teuchos::AnyNumberParameterEntryValidator;
  using Teuchos::Array;
  using Teuchos::ParameterList;
  using Teuchos::setIntParameter;
  using Teuchos::setDoubleParameter;
  using Teuchos::setStringToIntegralParameter;
  using Teuchos::tuple;
  using std::string;

  ParameterList dummy;
  ParameterList * PL = new ParameterList;

  // prevent Teuchos from converting parameter types
  AnyNumberParameterEntryValidator::AcceptedTypes intParam(false),
                                                  dblParam(false),
                                                  strParam(false);
  intParam.allowInt(true); 
  dblParam.allowDouble(true);
  strParam.allowString(true); 

  /* Allocate List for Smoothing Options */
# if defined(HAVE_PETSC) && defined(HAVE_ML_SUPERLU4_0)
  const int num_smoothers=30;
# elif defined(HAVE_PETSC) || defined(HAVE_ML_SUPERLU4_0)
  const int num_smoothers=29;
#elif defined(HAVE_ML_TekoSmoothers)
  const int num_smoothers=29; // won't work with SUPERLU or PETSC!
# else
  const int num_smoothers=28;
# endif
  const char* smoother_strings[num_smoothers]={"Aztec","IFPACK","Jacobi",
   "ML symmetric Gauss-Seidel","symmetric Gauss-Seidel","ML Gauss-Seidel",
   "Gauss-Seidel","block Gauss-Seidel","symmetric block Gauss-Seidel",
   "Chebyshev","MLS","Hiptmair","Amesos-KLU","Amesos-Superlu",
   "Amesos-UMFPACK","Amesos-Superludist","Amesos-MUMPS","user-defined",
   "SuperLU","IFPACK-Chebyshev","self","do-nothing","IC","ICT","ILU","ILUT",
   "Block Chebyshev","IFPACK-Block Chebyshev"
#  ifdef HAVE_PETSC
   ,"petsc"
#  endif
#  ifdef HAVE_ML_SUPERLU4_0
   ,"SILU"
//#else
//#error "No SuperLU for you!"					      
#  endif
#  ifdef HAVE_ML_TekoSmoothers
   ,"teko"
#  endif
   };
  Array<string> smoothers(num_smoothers);
  for(int i = 0; i<num_smoothers; i++) {
    smoothers[i] = smoother_strings[i];
  }

  /* General Options (Section 6.4.1) */
  setIntParameter("ML output",0,"Output Level",PL,intParam);
  setIntParameter("print unused",-2,"Print unused parameters",PL,intParam);
  setIntParameter("ML print initial list",-2,"Print initial list supplied to constructor",PL,intParam);
  setIntParameter("ML print final list",-2,"Print final list used by constructor",PL,intParam);
  setIntParameter("PDE equations",1,"# of PDE equations per node",PL,intParam);
  setStringToIntegralParameter<int>("eigen-analysis: type","cg","Scheme to compute spectral radius",
                               tuple<string>("cg","Anorm","power-method"),PL);
  setIntParameter("eigen-analysis: iterations",10,"# iterations of eigen-anaysis",PL,intParam);
  PL->set("ML label","dummy string");
  setIntParameter("print hierarchy",-2,"Print hierarchy.  0 or greater prints individual levels.",PL,intParam);

  /* Multigrid Cycle Options (Section 6.4.2) */
  setIntParameter("cycle applications",1,"# MG cycles",PL,intParam);
  setIntParameter("max levels",10,"Max # of levels",PL,intParam);
  setStringToIntegralParameter<int>("increasing or decreasing", "increasing", "Level numbering",tuple<string>("increasing","decreasing"),PL);
  setStringToIntegralParameter<int>("prec type", "MGV","Multigrid cycle type",tuple<string>("MGV","MGW","full-MGV","one-level-postsmoothing","two-level-additive","two-level-hybrid","two-level-hybrid2","projected MGV"),PL);
  PL->set("projected mode",(double**)0);
  setIntParameter("number of projected modes",0,"# of modes to be projected out before and after the V-cycle",PL,intParam);

  /* Aggregation and Prolongator Options (Section 6.4.3) */
  SetValidAggrParams(PL);
  for (int i = 0; i < 10; ++i) {
    char param[32];
    sprintf(param,"aggregation: list (level %d)",i); 
    SetValidAggrParams(&(PL->sublist(param)));
  }
  
  PL->set("energy minimization: enable",false);
  setIntParameter("energy minimization: type",2,"Norm to use for energy minimization",PL,intParam);  
  setDoubleParameter("energy minimization: droptol",0.0,"Drop tolerance for energy minimization",PL,dblParam);
  PL->set("energy minimization: cheap",false);

  /* Smoothing Options (Section 6.4.4) */
  SetValidSmooParams(PL,smoothers);
  for (int i = 0; i < 10; ++i) {
    char param[32];
    sprintf(param,"smoother: list (level %d)",i); 
    SetValidSmooParams(&(PL->sublist(param)),smoothers);
  }
  SetValidSmooParams(&(PL->sublist("coarse: list")),smoothers);

  /* Load-balancing Options (Section 6.4.6) */
  setIntParameter("repartition: enable",0,"Enable repartitioning",PL,intParam);
  setStringToIntegralParameter<int>("repartition: partitioner","Zoltan","Repartitioning method",tuple<string>("Zoltan","ParMETIS"),PL);
  setDoubleParameter("repartition: max min ratio",1.3,"Specifies desired maximum imbalance ratio",PL,dblParam);
  setIntParameter("repartition: min per proc",512,"Specifies minimum # rows / processor",PL,intParam);
  setIntParameter("repartition: put on single proc",5000,"Specifies max global problem to be put on one processor",PL,intParam);
  setDoubleParameter("repartition: node max min ratio",1.3,"Specifies desired maximum imbalance for nodal heirarchy (Maxwell)",PL,dblParam);
  setIntParameter("repartition: node min per proc",170,"Specifies minimum number of nodes per proc (Maxwell)",PL,intParam);
  setIntParameter("repartition: Zoltan dimensions",0,"Dimension of problem",PL,intParam);
  setIntParameter("repartition: start level",1,"Suppress repartitioning until this level",PL,intParam);

  /* Analysis Options (Section 6.4.7) */
  PL->set("analyze memory",false);
  PL->set("viz: enable",false);
  setStringToIntegralParameter<int>("viz: output format","vtk","Visualization format",tuple<string>("vtk","xyz","openx"),PL);
  PL->set("viz: print starting solution",false);
  setIntParameter("viz: equation to plot",-1,"Equation number to print",PL,intParam);

  /* Miscellaneous Options (Section 6.4.8) */
  PL->set("x-coordinates",(double*)0);
  PL->set("y-coordinates",(double*)0);
  PL->set("z-coordinates",(double*)0);
  PL->set("node: x-coordinates",(double*)0);
  PL->set("node: y-coordinates",(double*)0);
  PL->set("node: z-coordinates",(double*)0);
  PL->set("read XML",true); 
  PL->set("XML input file","ml_ParameterList.xml",string(""));

  /* Smoothed Aggregation and the Null Space (Section 6.4.9) */
  setStringToIntegralParameter<int>("null space: type","default vectors","Type of null space to use",tuple<string>("pre-computed","enriched","default vectors","elasticity from coordinates"),PL);
  PL->set("null space: vectors",(double*)0); 
  setIntParameter("null space: dimension",0,"Number of user-supplied null space vectors",PL,intParam);
  setIntParameter("null space: vectors to compute",1,"Number of vectors to compute",PL,intParam);
  PL->set("null space: add default vectors",true);

  /* Aggregation Strategies (Section 6.4.10) */
  PL->set("aggregation: aux: enable",false);
  setDoubleParameter("aggregation: aux: threshold",0.0,"Dropping threshold for auxillary matrix",PL,dblParam);  

  /* Unlisted Options */ 
  PL->set("ML debug mode",false);
  setStringToIntegralParameter<int>("default values","SA","Internal Option",tuple<string>("SA","DD","DD-ML","maxwell","NSSA","RefMaxwell","DD-ML-LU"),PL);
  PL->set("ML validate parameter list",true);
  setIntParameter("ML validate depth",0,"Internal option to control validation depth",PL,intParam);
  PL->set("ResetList",true); 
  setStringToIntegralParameter<int>("SetDefaults","not-set","Internal Option",tuple<string>("not-set","SA","DD","DD-ML","maxwell","NSSA","RefMaxwell"),PL);
  setIntParameter("ML node id",-1,"Experimental option to identify the processor node (vis-a-vis core) id",PL,intParam);

  /* Unlisted Options that should probably be listed */
  setIntParameter("aggregation: aux: max levels",10,"Unlisted option",PL,intParam);
  PL->set("low memory usage",false);
  setDoubleParameter("aggregation: edge prolongator drop threshold",0.0,"Unlisted option",PL,dblParam);
  PL->set("zero starting solution",true);
  PL->set("aggregation: block scaling",false);
  setIntParameter("profile: operator iterations",0,"Unlisted option",PL,intParam);
  setDoubleParameter("subsmoother: edge alpha",20.0,"alpha for edge Chebyshev polynomial in Hiptmair",PL,dblParam); 
  setDoubleParameter("subsmoother: node alpha",20.0,"alpha for node Chebyshev polynomial in Hiptmair",PL,dblParam); 
  PL->set("reuse: enable",false);
  
  /* Unlisted options that should probably go away */
  setIntParameter("output",0,"Output Level",PL,intParam);

  /* Hightly experimental */
  PL->set("repartition: output timings",false);
  setIntParameter("repartition: estimated iterations",0,"Estimated number of iterations",PL,intParam);
  setStringToIntegralParameter<int>("repartition: Zoltan type","RCB","Type of repartitioner to use",tuple<string>("RCB","hypergraph","fast hypergraph"),PL);

  /* EXPERIMENTAL - Half-GS Smoothing */
  PL->set("smoother: Gauss-Seidel efficient symmetric",false); 
  
  /* Coarse IFPACK Solvers - experimental */
  PL->set("coarse: ifpack list",dummy);
  PL->set("coarse: ifpack type",std::string(""));
  setIntParameter("coarse: ifpack overlap",0,"Unlisted option",PL,intParam);
  setDoubleParameter("coarse: ifpack level-of-fill",0.0,"Unlisted option",PL,dblParam);
  setDoubleParameter("coarse: ifpack relative threshold",1.0,"Unlisted option",PL,dblParam);
  setDoubleParameter("coarse: ifpack absolute threshold",0.0,"Unlisted option",PL,dblParam);

  /* EXPERIMENTAL - RefMaxwell block parallelization */
  PL->set("partitioner: options",dummy);  
  PL->sublist("partitioner: options").disableRecursiveValidation();

  /* EXPERIMENTAL - node aware code */
  setIntParameter("ML node id", -1, "Unlisted option", PL, intParam);

  return PL;
}
Exemple #22
0
  void ParameterListInterpreter<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::UpdateFactoryManager(Teuchos::ParameterList& paramList,
        const Teuchos::ParameterList& defaultList, FactoryManager& manager) const {
    // NOTE: Factory::SetParameterList must be called prior to Factory::SetFactory, as
    // SetParameterList sets default values for non mentioned parameters, including factories

    // === Smoothing ===
    bool isCustomSmoother =
        paramList.isParameter("smoother: pre or post") ||
        paramList.isParameter("smoother: type")    || paramList.isParameter("smoother: pre type")    || paramList.isParameter("smoother: post type")   ||
        paramList.isSublist  ("smoother: params")  || paramList.isSublist  ("smoother: pre params")  || paramList.isSublist  ("smoother: post params") ||
        paramList.isParameter("smoother: sweeps")  || paramList.isParameter("smoother: pre sweeps")  || paramList.isParameter("smoother: post sweeps") ||
        paramList.isParameter("smoother: overlap") || paramList.isParameter("smoother: pre overlap") || paramList.isParameter("smoother: post overlap");;
    MUELU_READ_2LIST_PARAM(paramList, defaultList, "smoother: pre or post", std::string, "both", PreOrPost);
    if (PreOrPost == "none") {
      manager.SetFactory("Smoother", Teuchos::null);

    } else if (isCustomSmoother) {
      // FIXME: get default values from the factory
      // NOTE: none of the smoothers at the moment use parameter validation framework, so we
      // cannot get the default values from it.
#define TEST_MUTUALLY_EXCLUSIVE(arg1,arg2) \
      TEUCHOS_TEST_FOR_EXCEPTION(paramList.isParameter(#arg1) && paramList.isParameter(#arg2), \
                                 Exceptions::InvalidArgument, "You cannot specify both \""#arg1"\" and \""#arg2"\"");
#define TEST_MUTUALLY_EXCLUSIVE_S(arg1,arg2) \
      TEUCHOS_TEST_FOR_EXCEPTION(paramList.isSublist(#arg1) && paramList.isSublist(#arg2), \
                                 Exceptions::InvalidArgument, "You cannot specify both \""#arg1"\" and \""#arg2"\"");

      TEST_MUTUALLY_EXCLUSIVE  ("smoother: type",    "smoother: pre type");
      TEST_MUTUALLY_EXCLUSIVE  ("smoother: type",    "smoother: post type");
      TEST_MUTUALLY_EXCLUSIVE  ("smoother: sweeps",  "smoother: pre sweeps");
      TEST_MUTUALLY_EXCLUSIVE  ("smoother: sweeps",  "smoother: post sweeps");
      TEST_MUTUALLY_EXCLUSIVE  ("smoother: overlap", "smoother: pre overlap");
      TEST_MUTUALLY_EXCLUSIVE  ("smoother: overlap", "smoother: post overlap");
      TEST_MUTUALLY_EXCLUSIVE_S("smoother: params",  "smoother: pre params");
      TEST_MUTUALLY_EXCLUSIVE_S("smoother: params",  "smoother: post params");
      TEUCHOS_TEST_FOR_EXCEPTION(PreOrPost == "both" && (paramList.isParameter("smoother: pre type") != paramList.isParameter("smoother: post type")),
                                 Exceptions::InvalidArgument, "You must specify both \"smoother: pre type\" and \"smoother: post type\"");

      // Default values
      int overlap = 0;
      ParameterList defaultSmootherParams;
      defaultSmootherParams.set("relaxation: type",           "Symmetric Gauss-Seidel");
      defaultSmootherParams.set("relaxation: sweeps",         Teuchos::OrdinalTraits<LO>::one());
      defaultSmootherParams.set("relaxation: damping factor", Teuchos::ScalarTraits<Scalar>::one());

      RCP<SmootherPrototype> preSmoother = Teuchos::null, postSmoother = Teuchos::null;
      std::string            preSmootherType,             postSmootherType;
      ParameterList          preSmootherParams,           postSmootherParams;

      if (paramList.isParameter("smoother: overlap"))
        overlap = paramList.get<int>("smoother: overlap");

      if (PreOrPost == "pre" || PreOrPost == "both") {
        if (paramList.isParameter("smoother: pre type")) {
          preSmootherType = paramList.get<std::string>("smoother: pre type");
        } else {
          MUELU_READ_2LIST_PARAM(paramList, defaultList, "smoother: type", std::string, "RELAXATION", preSmootherTypeTmp);
          preSmootherType = preSmootherTypeTmp;
        }
        if (paramList.isParameter("smoother: pre overlap"))
          overlap = paramList.get<int>("smoother: pre overlap");

        if (paramList.isSublist("smoother: pre params"))
          preSmootherParams = paramList.sublist("smoother: pre params");
        else if (paramList.isSublist("smoother: params"))
          preSmootherParams = paramList.sublist("smoother: params");
        else if (defaultList.isSublist("smoother: params"))
          preSmootherParams = defaultList.sublist("smoother: params");
        else if (preSmootherType == "RELAXATION")
          preSmootherParams = defaultSmootherParams;

        preSmoother = rcp(new TrilinosSmoother(preSmootherType, preSmootherParams, overlap));
      }

      if (PreOrPost == "post" || PreOrPost == "both") {
        if (paramList.isParameter("smoother: post type"))
          postSmootherType = paramList.get<std::string>("smoother: post type");
        else {
          MUELU_READ_2LIST_PARAM(paramList, defaultList, "smoother: type", std::string, "RELAXATION", postSmootherTypeTmp);
          postSmootherType = postSmootherTypeTmp;
        }

        if (paramList.isSublist("smoother: post params"))
          postSmootherParams = paramList.sublist("smoother: post params");
        else if (paramList.isSublist("smoother: params"))
          postSmootherParams = paramList.sublist("smoother: params");
        else if (defaultList.isSublist("smoother: params"))
          postSmootherParams = defaultList.sublist("smoother: params");
        else if (postSmootherType == "RELAXATION")
          postSmootherParams = defaultSmootherParams;
        if (paramList.isParameter("smoother: post overlap"))
          overlap = paramList.get<int>("smoother: post overlap");

        if (postSmootherType == preSmootherType && areSame(preSmootherParams, postSmootherParams))
          postSmoother = preSmoother;
        else
          postSmoother = rcp(new TrilinosSmoother(postSmootherType, postSmootherParams, overlap));
      }

      manager.SetFactory("Smoother", rcp(new SmootherFactory(preSmoother, postSmoother)));
    }

    // === Coarse solver ===
    bool isCustomCoarseSolver =
        paramList.isParameter("coarse: type")   ||
        paramList.isParameter("coarse: params");
    if (paramList.isParameter("coarse: type") && paramList.get<std::string>("coarse: type") == "none") {
      manager.SetFactory("CoarseSolver", Teuchos::null);

    } else if (isCustomCoarseSolver) {
      // FIXME: get default values from the factory
      // NOTE: none of the smoothers at the moment use parameter validation framework, so we
      // cannot get the default values from it.
      MUELU_READ_2LIST_PARAM(paramList, defaultList, "coarse: type", std::string, "", coarseType);

      ParameterList coarseParams;
      if (paramList.isSublist("coarse: params"))
        coarseParams = paramList.sublist("coarse: params");
      else if (defaultList.isSublist("coarse: params"))
        coarseParams = defaultList.sublist("coarse: params");

      RCP<SmootherPrototype> coarseSmoother;
      // TODO: this is not a proper place to check. If we consider direct solver to be a special
      // case of smoother, we would like to unify Amesos and Ifpack2 smoothers in src/Smoothers, and
      // have a single factory responsible for those. Then, this check would belong there.
      if (coarseType == "RELAXATION" || coarseType == "CHEBYSHEV" ||
          coarseType == "ILUT" || coarseType == "ILU" || coarseType == "RILUK" || coarseType == "SCHWARZ")
        coarseSmoother = rcp(new TrilinosSmoother(coarseType, coarseParams));
      else
        coarseSmoother = rcp(new DirectSolver(coarseType, coarseParams));

      manager.SetFactory("CoarseSolver", rcp(new SmootherFactory(coarseSmoother)));
    }

    // === Aggregation ===
    // Aggregation graph
    RCP<CoalesceDropFactory> dropFactory = rcp(new CoalesceDropFactory());
    ParameterList dropParams;
    dropParams.set("lightweight wrap", true);
    MUELU_TEST_AND_SET_PARAM(dropParams, "algorithm",                     paramList, defaultList, "aggregation: drop scheme",         std::string);
    // Rename classical to original
    if (dropParams.isParameter("algorithm") && dropParams.get<std::string>("algorithm") == "classical")
      dropParams.set("algorithm", "original");
    MUELU_TEST_AND_SET_PARAM(dropParams, "aggregation threshold",         paramList, defaultList, "aggregation: drop tol",            double);
    MUELU_TEST_AND_SET_PARAM(dropParams, "Dirichlet detection threshold", paramList, defaultList, "aggregation: Dirichlet threshold", double);

    dropFactory->SetParameterList(dropParams);
    manager.SetFactory("Graph", dropFactory);

    // Aggregation sheme
    MUELU_READ_2LIST_PARAM(paramList, defaultList, "aggregation: type", std::string, "uncoupled", aggType);
    RCP<Factory> aggFactory;
    if      (aggType == "uncoupled") {
      aggFactory = rcp(new UncoupledAggregationFactory());
      ParameterList aggParams;
      MUELU_TEST_AND_SET_PARAM(aggParams, "mode",                 paramList, defaultList, "aggregation: mode",          std::string);
      MUELU_TEST_AND_SET_PARAM(aggParams, "MinNodesPerAggregate", paramList, defaultList, "aggregation: min agg size",  int);
      MUELU_TEST_AND_SET_PARAM(aggParams, "MaxNodesPerAggregate", paramList, defaultList, "aggregation: max agg size",  int);
      MUELU_TEST_AND_SET_PARAM(aggParams, "aggregation: preserve Dirichlet points", paramList, defaultList, "aggregation: preserve Dirichlet points", bool);
      aggFactory->SetParameterList(aggParams);

    } else if (aggType == "coupled") {
Exemple #23
0
  TEUCHOS_UNIT_TEST(RAPFactory, ImplicitTranspose)
  {
    out << "version: " << MueLu::Version() << std::endl;

    RCP<const Teuchos::Comm<int> > comm = Parameters::getDefaultComm();

    if (comm->getSize() > 1 && TestHelpers::Parameters::getLib() == Xpetra::UseEpetra ) {
      out << "Skipping ImplicitTranspose test for Epetra and #proc>1" << std::endl;
      return;
    }

    // build test-specific default factory manager
    RCP<FactoryManager> defManager = rcp(new FactoryManager());
    defManager->SetFactory("A", rcp(MueLu::NoFactory::get(),false));         // dummy factory for A
    defManager->SetFactory("Nullspace", rcp(new NullspaceFactory()));        // real null space factory for Ptent
    defManager->SetFactory("Graph", rcp(new CoalesceDropFactory()));         // real graph factory for Ptent
    defManager->SetFactory("Aggregates", rcp(new CoupledAggregationFactory()));   // real aggregation factory for Ptent

    Level fineLevel, coarseLevel;
    TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::createTwoLevelHierarchy(fineLevel, coarseLevel);

    // overwrite default factory manager
    fineLevel.SetFactoryManager(defManager);
    coarseLevel.SetFactoryManager(defManager);

    RCP<Matrix> Op = TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::Build1DPoisson(19*comm->getSize());
    fineLevel.Set("A",Op);

    TentativePFactory tentpFactory;
    SaPFactory sapFactory;
    sapFactory.SetFactory("P",rcpFromRef(tentpFactory));
    TransPFactory transPFactory;
    transPFactory.SetFactory("P", rcpFromRef(sapFactory));
    coarseLevel.Request("P", &sapFactory);
    coarseLevel.Request("R", &transPFactory);

    coarseLevel.Request(sapFactory);
    coarseLevel.Request(transPFactory);
    sapFactory.Build(fineLevel, coarseLevel);
    transPFactory.Build(fineLevel,coarseLevel);
    RAPFactory rap;
    ParameterList rapList = *(rap.GetValidParameterList());
    rapList.set("transpose: use implicit", true);
    rap.SetParameterList(rapList);
    rap.SetFactory("P", rcpFromRef(sapFactory));
    rap.SetFactory("R", rcpFromRef(transPFactory));
    coarseLevel.Request("A", &rap);

    coarseLevel.Request(rap);
    rap.Build(fineLevel,coarseLevel);

    RCP<Matrix> A = fineLevel.Get< RCP<Matrix> >("A");
    RCP<Matrix> P = coarseLevel.Get< RCP<Matrix> >("P", &sapFactory);
    RCP<Matrix> R = coarseLevel.Get< RCP<Matrix> >("R", &transPFactory);

    //std::string filename = "A.dat";
    //Utils::Write(filename,Op);
    //filename = "P.dat";
    //Utils::Write(filename,P);

    RCP<MultiVector> workVec1 = MultiVectorFactory::Build(P->getRangeMap(),1);
    RCP<MultiVector> workVec2 = MultiVectorFactory::Build(Op->getRangeMap(),1);
    RCP<MultiVector> result1 = MultiVectorFactory::Build(P->getDomainMap(),1);
    RCP<MultiVector> X = MultiVectorFactory::Build(P->getDomainMap(),1);
    X->randomize();
    //out.precision(12);
    //out.setOutputToRootOnly(-1);
    //X->describe(out,Teuchos::VERB_EXTREME);

    //Calculate result1 = P^T*(A*(P*X))
    P->apply(*X,*workVec1,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
    Op->apply(*workVec1,*workVec2,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
    P->apply(*workVec2,*result1,Teuchos::TRANS,(SC)1.0,(SC)0.0);

    RCP<Matrix> coarseOp = coarseLevel.Get< RCP<Matrix> >("A", &rap);

    //Calculate result2 = (R*A*P)*X
    RCP<MultiVector> result2 = MultiVectorFactory::Build(P->getDomainMap(),1);
    coarseOp->apply(*X,*result2,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);

    Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> normX(1), normResult1(1),normResult2(1);
    X->norm2(normX);
    out << "This test checks the correctness of the Galerkin triple "
        << "matrix product by comparing (RAP)*X to R(A(P*X)), where R is the implicit transpose of P." << std::endl;
    out << "||X||_2 = " << normX << std::endl;
    result1->norm2(normResult1);
    result2->norm2(normResult2);
    TEST_FLOATING_EQUALITY(normResult1[0], normResult2[0], 1e-12);

  } // Correctness test
Exemple #24
0
int main(int argc, char *argv[]) {
  typedef double MeshScalar;
  typedef double BasisScalar;
  typedef Tpetra::DefaultPlatform::DefaultPlatformType::NodeType Node;
  typedef Teuchos::ScalarTraits<Scalar>::magnitudeType magnitudeType;

  //double g_mean_exp = 1.906587e-01;      // expected response mean
  //double g_std_dev_exp = 8.680605e-02;  // expected response std. dev.
  //double g_tol = 1e-6;               // tolerance on determining success



  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::Array;
  using Teuchos::ArrayRCP;
  using Teuchos::ArrayView;
  using Teuchos::ParameterList;

// Initialize MPI
#ifdef HAVE_MPI
  MPI_Init(&argc,&argv);
#endif

//  feenableexcept(FE_ALL_EXCEPT);

  LocalOrdinal MyPID;

  try {

    // Create a communicator for Epetra objects
    RCP<const Epetra_Comm> globalComm;
#ifdef HAVE_MPI
    globalComm = rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
#else
    globalComm = rcp(new Epetra_SerialComm);
#endif
    MyPID = globalComm->MyPID();

    // Setup command line options
    Teuchos::CommandLineProcessor CLP;
    CLP.setDocString(
      "This example runs an interlaced stochastic Galerkin solvers.\n");

    int n = 32;
    CLP.setOption("num_mesh", &n, "Number of mesh points in each direction");

    // multigrid specific options
    int minAggSize = 1;
    CLP.setOption("min_agg_size", &minAggSize, "multigrid aggregate size");
    int smootherSweeps = 3;
    CLP.setOption("smoother_sweeps", &smootherSweeps, "# multigrid smoother sweeps");
    int plainAgg=1;
    CLP.setOption("plain_aggregation", &plainAgg, "plain aggregation");
    LocalOrdinal nsSize=-1;
    CLP.setOption("nullspace_size", &nsSize, "nullspace dimension");


    bool symmetric = false;
    CLP.setOption("symmetric", "unsymmetric", &symmetric, 
                  "Symmetric discretization");

    int num_spatial_procs = -1;
    CLP.setOption("num_spatial_procs", &num_spatial_procs, "Number of spatial processors (set -1 for all available procs)");

    SG_RF randField = UNIFORM;
    CLP.setOption("rand_field", &randField, 
                  num_sg_rf, sg_rf_values, sg_rf_names,
                  "Random field type");

    double mu = 0.2;
    CLP.setOption("mean", &mu, "Mean");

    double s = 0.1;
    CLP.setOption("std_dev", &s, "Standard deviation");

    int num_KL = 2;
    CLP.setOption("num_kl", &num_KL, "Number of KL terms");

    int order = 3;
    CLP.setOption("order", &order, "Polynomial order");

    bool normalize_basis = true;
    CLP.setOption("normalize", "unnormalize", &normalize_basis, 
                  "Normalize PC basis");

    Krylov_Method solver_method = GMRES;
    CLP.setOption("solver_method", &solver_method, 
                  num_krylov_method, krylov_method_values, krylov_method_names, 
                  "Krylov solver method");

    SG_Prec prec_method = STOCHASTIC;
    CLP.setOption("prec_method", &prec_method, 
                  num_sg_prec, sg_prec_values, sg_prec_names,
                  "Preconditioner method");

    SG_Div division_method = DIRECT;
    CLP.setOption("division_method", &division_method, 
                  num_sg_div, sg_div_values, sg_div_names,
                  "Stochastic division method");

    SG_DivPrec divprec_method = NO;
    CLP.setOption("divprec_method", &divprec_method,
                  num_sg_divprec, sg_divprec_values, sg_divprec_names,
                  "Preconditioner for division method");
    Schur_option schur_option = diag;
    CLP.setOption("schur_option", &schur_option,
                  num_schur_option, Schur_option_values, schur_option_names,
                  "Schur option");
    Prec_option prec_option = whole;
    CLP.setOption("prec_option", &prec_option,
                  num_prec_option, Prec_option_values, prec_option_names,
                  "Prec option");


    double solver_tol = 1e-12;
    CLP.setOption("solver_tol", &solver_tol, "Outer solver tolerance");

    double div_tol = 1e-6;
    CLP.setOption("div_tol", &div_tol, "Tolerance in Iterative Solver");
    
    int prec_level = 1;
    CLP.setOption("prec_level", &prec_level, "Level in Schur Complement Prec 0->Solve A0u0=g0 with division; 1->Form 1x1 Schur Complement");

    int max_it_div = 50;
    CLP.setOption("max_it_div", &max_it_div, "Maximum # of Iterations in Iterative Solver for Division");

    bool equilibrate = true; //JJH 8/26/12 changing to true to match ETP example
    CLP.setOption("equilibrate", "noequilibrate", &equilibrate,
                  "Equilibrate the linear system");


    CLP.parse( argc, argv );

    if (MyPID == 0) {
      std::cout << "Summary of command line options:" << std::endl
                << "\tnum_mesh           = " << n << std::endl
                << "\tsymmetric          = " << symmetric << std::endl
                << "\tnum_spatial_procs  = " << num_spatial_procs << std::endl
                << "\trand_field         = " << sg_rf_names[randField] 
                << std::endl
                << "\tmean               = " << mu << std::endl
                << "\tstd_dev            = " << s << std::endl
                << "\tnum_kl             = " << num_KL << std::endl
                << "\torder              = " << order << std::endl
                << "\tnormalize_basis    = " << normalize_basis << std::endl
                << "\tsolver_method      = " << krylov_method_names[solver_method] << std::endl
                << "\tprec_method        = " << sg_prec_names[prec_method]    << std::endl
                << "\tdivision_method    = " << sg_div_names[division_method]     << std::endl
                << "\tdiv_tol            = " << div_tol << std::endl
                << "\tdiv_prec           = " << sg_divprec_names[divprec_method]      << std::endl
                << "\tprec_level         = " << prec_level << std::endl
                << "\tmax_it_div     = " << max_it_div << std::endl;
    }
    bool nonlinear_expansion = false;
    if (randField == UNIFORM)
      nonlinear_expansion = false;
    else if (randField == LOGNORMAL)
      nonlinear_expansion = true;

    {
    TEUCHOS_FUNC_TIME_MONITOR("Total PCE Calculation Time");

    // Create Stochastic Galerkin basis and expansion
    Teuchos::Array< RCP<const Stokhos::OneDOrthogPolyBasis<LocalOrdinal,BasisScalar> > > bases(num_KL); 
    for (LocalOrdinal i=0; i<num_KL; i++)
      if (randField == UNIFORM)
        bases[i] = rcp(new Stokhos::LegendreBasis<LocalOrdinal,BasisScalar>(order, normalize_basis));
      else if (randField == LOGNORMAL)
        bases[i] = rcp(new Stokhos::HermiteBasis<int,double>(order, normalize_basis));
    RCP<const Stokhos::CompletePolynomialBasis<LocalOrdinal,BasisScalar> > basis = 
      rcp(new Stokhos::CompletePolynomialBasis<LocalOrdinal,BasisScalar>(bases, 1e-12));
    LocalOrdinal sz = basis->size();
    RCP<Stokhos::Sparse3Tensor<LocalOrdinal,BasisScalar> > Cijk = 
      basis->computeTripleProductTensor(sz);
    RCP<const Stokhos::Quadrature<int,double> > quad = 
      rcp(new Stokhos::TensorProductQuadrature<int,double>(basis));
    RCP<ParameterList> expn_params = Teuchos::rcp(new ParameterList);
    if (division_method == MEAN_DIV) {
      expn_params->set("Division Strategy", "Mean-Based");
      expn_params->set("Use Quadrature for Division", false);
    }
    else if (division_method == DIRECT) {
      expn_params->set("Division Strategy", "Dense Direct");
      expn_params->set("Use Quadrature for Division", false);
    }
    else if (division_method == SPD_DIRECT) {
      expn_params->set("Division Strategy", "SPD Dense Direct");
      expn_params->set("Use Quadrature for Division", false);
    }
    else if (division_method == CGD) {
      expn_params->set("Division Strategy", "CG");
      expn_params->set("Use Quadrature for Division", false);
    }

    else if (division_method == QUAD) {
      expn_params->set("Use Quadrature for Division", true);
    }

    if (divprec_method == NO)
         expn_params->set("Prec Strategy", "None");
    else if (divprec_method == DIAG)
         expn_params->set("Prec Strategy", "Diag");
    else if (divprec_method == JACOBI)
         expn_params->set("Prec Strategy", "Jacobi");
    else if (divprec_method == GS)
         expn_params->set("Prec Strategy", "GS");
    else if (divprec_method == SCHUR)
         expn_params->set("Prec Strategy", "Schur");

    if (schur_option == diag)
        expn_params->set("Schur option", "diag");
    else
        expn_params->set("Schur option", "full");
    if (prec_option == linear)
        expn_params->set("Prec option", "linear");


    if (equilibrate)
      expn_params->set("Equilibrate", 1);
    else
      expn_params->set("Equilibrate", 0); 
    expn_params->set("Division Tolerance", div_tol);
    expn_params->set("prec_iter", prec_level);
    expn_params->set("max_it_div", max_it_div);

    RCP<Stokhos::OrthogPolyExpansion<LocalOrdinal,BasisScalar> > expansion = 
      rcp(new Stokhos::QuadOrthogPolyExpansion<LocalOrdinal,BasisScalar>(
            basis, Cijk, quad, expn_params));

    if (MyPID == 0)
      std::cout << "Stochastic Galerkin expansion size = " << sz << std::endl;

    // Create stochastic parallel distribution
    ParameterList parallelParams;
    parallelParams.set("Number of Spatial Processors", num_spatial_procs);
    // parallelParams.set("Rebalance Stochastic Graph", true);
    // Teuchos::ParameterList& isorropia_params = 
    //   parallelParams.sublist("Isorropia");
    // isorropia_params.set("Balance objective", "nonzeros");
    RCP<Stokhos::ParallelData> sg_parallel_data =
      rcp(new Stokhos::ParallelData(basis, Cijk, globalComm, parallelParams));
    RCP<const EpetraExt::MultiComm> sg_comm = 
      sg_parallel_data->getMultiComm();
    RCP<const Epetra_Comm> app_comm = 
      sg_parallel_data->getSpatialComm();

    // Create Teuchos::Comm from Epetra_Comm
    RCP< Teuchos::Comm<int> > teuchos_app_comm;
#ifdef HAVE_MPI
    RCP<const Epetra_MpiComm> app_mpi_comm = 
      Teuchos::rcp_dynamic_cast<const Epetra_MpiComm>(app_comm);
    RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > raw_mpi_comm = 
      Teuchos::opaqueWrapper(app_mpi_comm->Comm());
    teuchos_app_comm = rcp(new Teuchos::MpiComm<int>(raw_mpi_comm));
#else
    teuchos_app_comm = rcp(new Teuchos::SerialComm<int>());
#endif

    // Create application
    typedef twoD_diffusion_problem<Scalar,MeshScalar,BasisScalar,LocalOrdinal,GlobalOrdinal,Node> problem_type;
    RCP<problem_type> model = 
      rcp(new problem_type(teuchos_app_comm, n, num_KL, s, mu, 
               nonlinear_expansion, symmetric));

    // Create vectors and operators
    typedef problem_type::Tpetra_Vector Tpetra_Vector;
    typedef problem_type::Tpetra_CrsMatrix Tpetra_CrsMatrix;
    typedef Tpetra::MatrixMarket::Writer<Tpetra_CrsMatrix> Writer;
    //Xpetra matrices
    typedef Xpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_CrsMatrix;
    typedef Xpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> Xpetra_MultiVector;
    typedef Xpetra::MultiVectorFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node> Xpetra_MultiVectorFactory;
    typedef Xpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_Operator;
    typedef Xpetra::TpetraCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_TpetraCrsMatrix;
    typedef Xpetra::CrsOperator<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_CrsOperator;
    typedef Belos::MueLuOp<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Belos_MueLuOperator;
    //MueLu typedefs
    typedef MueLu::Hierarchy<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> MueLu_Hierarchy;
    typedef MueLu::SmootherPrototype<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> SmootherPrototype;
    typedef MueLu::TrilinosSmoother<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> TrilinosSmoother;
    typedef MueLu::SmootherFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> SmootherFactory;
    typedef MueLu::FactoryManager<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> FactoryManager;

    RCP<Tpetra_Vector> p = Tpetra::createVector<Scalar>(model->get_p_map(0));
    RCP<Tpetra_Vector> x = Tpetra::createVector<Scalar>(model->get_x_map());
    x->putScalar(0.0);
    RCP<Tpetra_Vector> f = Tpetra::createVector<Scalar>(model->get_f_map());
    RCP<Tpetra_Vector> dx = Tpetra::createVector<Scalar>(model->get_x_map());
    RCP<Tpetra_CrsMatrix> J = model->create_W();
    RCP<Tpetra_CrsMatrix> J0;
    if (prec_method == MEAN)
      J0 = model->create_W();

    // Set PCE expansion of p
    p->putScalar(0.0);
    ArrayRCP<Scalar> p_view = p->get1dViewNonConst();
    for (ArrayRCP<Scalar>::size_type i=0; i<p_view.size(); i++) {
      p_view[i].reset(expansion);
      p_view[i].copyForWrite();
    }
    Array<double> point(num_KL, 1.0);
    Array<double> basis_vals(sz);
    basis->evaluateBases(point, basis_vals);
    if (order > 0) {
      for (int i=0; i<num_KL; i++) {
        p_view[i].term(i,1) = 1.0 / basis_vals[i+1];
      }
    }

    // Create preconditioner
    typedef Ifpack2::Preconditioner<Scalar,LocalOrdinal,GlobalOrdinal,Node> Tprec;
    RCP<Belos_MueLuOperator> M;
    RCP<MueLu_Hierarchy> H;
    RCP<Xpetra_CrsMatrix> xcrsJ = rcp(new Xpetra_TpetraCrsMatrix(J));
    RCP<Xpetra_Operator> xopJ = rcp(new Xpetra_CrsOperator(xcrsJ));
    if (prec_method != NONE) {
      ParameterList precParams;
      std::string prec_name = "RILUK";
      precParams.set("fact: iluk level-of-fill", 1);
      precParams.set("fact: iluk level-of-overlap", 0);
      //Ifpack2::Factory factory;
      RCP<Xpetra_Operator> xopJ0;
      if (prec_method == MEAN) {
        RCP<Xpetra_CrsMatrix> xcrsJ0 = rcp(new Xpetra_TpetraCrsMatrix(J0));
        xopJ0 = rcp(new Xpetra_CrsOperator(xcrsJ0));
        //M = factory.create<Tpetra_CrsMatrix>(prec_name, J0);
      } else if (prec_method == STOCHASTIC) {
        xopJ0 = xopJ;
        //M = factory.create<Tpetra_CrsMatrix>(prec_name, J);
      }
      H = rcp(new MueLu_Hierarchy(xopJ0));
      M = rcp(new Belos_MueLuOperator(H));
      //M->setParameters(precParams);
      if (nsSize!=-1) sz=nsSize;
      RCP<Xpetra_MultiVector> Z = Xpetra_MultiVectorFactory::Build(xcrsJ->getDomainMap(), sz);
      size_t n = Z->getLocalLength();
      for (LocalOrdinal j=0; j<sz; ++j) {
        ArrayRCP<Scalar> col = Z->getDataNonConst(j);
        for (size_t i=0; i<n; ++i) {
          col[i].reset(expansion);
          col[i].copyForWrite();
          col[i].fastAccessCoeff(j) = 1.0;
        }
      }
      H->GetLevel(0)->Set("Nullspace", Z);
      //RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
      //fos->setOutputToRootOnly(-1);
      //Z->describe(*fos);
    }

    // Evaluate model
    model->computeResidual(*x, *p, *f);
    model->computeJacobian(*x, *p, *J);

    // Compute mean for mean-based preconditioner
    if (prec_method == MEAN) {
      size_t nrows = J->getNodeNumRows();
      ArrayView<const LocalOrdinal> indices;
      ArrayView<const Scalar> values;
      J0->resumeFill();
      for (size_t i=0; i<nrows; i++) {
        J->getLocalRowView(i, indices, values);
        Array<Scalar> values0(values.size());
        for (LocalOrdinal j=0; j<values.size(); j++)
          values0[j] = values[j].coeff(0);
        J0->replaceLocalValues(i, indices, values0);
      }
      J0->fillComplete();
    }

    // compute preconditioner
    if (prec_method != NONE) {
      //M->initialize();
      //M->compute();

      //override MueLu defaults via factory manager
      RCP<FactoryManager> fm = rcp( new FactoryManager() );;

      //smoother
      ParameterList smootherParamList;
      /*
      smootherParamList.set("chebyshev: degree", smootherSweeps);
      smootherParamList.set("chebyshev: ratio eigenvalue", (double) 20);
      smootherParamList.set("chebyshev: max eigenvalue", (double) -1.0);
      smootherParamList.set("chebyshev: min eigenvalue", (double) 1.0);
      smootherParamList.set("chebyshev: zero starting solution", true);
      RCP<SmootherPrototype> smooPrototype     = rcp( new TrilinosSmoother("CHEBYSHEV", smootherParamList) );
      */
      smootherParamList.set("relaxation: sweeps", smootherSweeps);
      smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
      RCP<SmootherPrototype> smooPrototype     = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );

      RCP<SmootherFactory>   smooFact      = rcp( new SmootherFactory(smooPrototype) );
      fm->SetFactory("Smoother", smooFact);

      // coarse level solve
      ParameterList coarseParamList;
      coarseParamList.set("fact: level-of-fill", 0);
      RCP<SmootherPrototype> coarsePrototype     = rcp( new TrilinosSmoother("ILUT", coarseParamList) );
      RCP<SmootherFactory>   coarseSolverFact      = rcp( new SmootherFactory(coarsePrototype, Teuchos::null) );
      fm->SetFactory("CoarseSolver", coarseSolverFact);

      //allow for larger aggregates
      typedef MueLu::UCAggregationFactory<LocalOrdinal,GlobalOrdinal,Node,LocalMatOps>
      MueLu_UCAggregationFactory;
      RCP<MueLu_UCAggregationFactory> aggFact = rcp(new MueLu_UCAggregationFactory());
      aggFact->SetMinNodesPerAggregate(minAggSize);
      fm->SetFactory("Aggregates", aggFact);

      //turn off damping
      typedef MueLu::SaPFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> MueLu_SaPFactory;
      if (plainAgg) {
        RCP<MueLu_SaPFactory> sapFactory = rcp(new MueLu_SaPFactory);
        sapFactory->SetDampingFactor( (Scalar) 0.0 );
        fm->SetFactory("P", sapFactory);
      }

      H->Setup(*fm);
    }

    // Setup Belos solver
    RCP<ParameterList> belosParams = rcp(new ParameterList);
   


    belosParams->set("Flexible Gmres", false);

    belosParams->set("Num Blocks", 500);//20
    belosParams->set("Convergence Tolerance", solver_tol);
    belosParams->set("Maximum Iterations", 1000);
    belosParams->set("Verbosity", 33);
    belosParams->set("Output Style", 1);
    belosParams->set("Output Frequency", 1);
    typedef Tpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> MV;
    typedef Belos::OperatorT<Tpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> > OP;
    typedef Belos::OperatorTraits<Scalar,MV,OP> BOPT;
    typedef Belos::MultiVecTraits<Scalar,MV> BMVT;
    typedef Belos::MultiVecTraits<double,MV> BTMVT;
    typedef Belos::LinearProblem<double,MV,OP> BLinProb;
    typedef Belos::XpetraOp<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> BXpetraOp;
    RCP<OP> belosJ = rcp(new BXpetraOp(xopJ)); // Turns an Xpetra::Operator object into a Belos operator
    RCP< BLinProb > problem = rcp(new BLinProb(belosJ, dx, f));
    if (prec_method != NONE)
      problem->setRightPrec(M);
    problem->setProblem();
    RCP<Belos::SolverManager<double,MV,OP> > solver;
    if (solver_method == CG)
      solver = rcp(new Belos::PseudoBlockCGSolMgr<double,MV,OP>(problem, belosParams));
    else if (solver_method == GMRES)
      solver = rcp(new Belos::BlockGmresSolMgr<double,MV,OP>(problem, belosParams));
    

    // Print initial residual norm
    std::vector<double> norm_f(1);
    //BMVT::MvNorm(*f, norm_f);
    BTMVT::MvNorm(*f, norm_f);
    if (MyPID == 0)
      std::cout << "\nInitial residual norm = " << norm_f[0] << std::endl;

    // Solve linear system
    Belos::ReturnType ret = solver->solve();

    if (MyPID == 0) {
      if (ret == Belos::Converged)
        std::cout << "Solver converged!" << std::endl;
      else
        std::cout << "Solver failed to converge!" << std::endl;
    }

    // Update x
    x->update(-1.0, *dx, 1.0);
    Writer::writeDenseFile("stochastic_solution.mm", x);

    // Compute new residual & response function
    RCP<Tpetra_Vector> g = Tpetra::createVector<Scalar>(model->get_g_map(0));
    f->putScalar(0.0);
    model->computeResidual(*x, *p, *f);
    model->computeResponse(*x, *p, *g);

    // Print final residual norm
    //BMVT::MvNorm(*f, norm_f);
    BTMVT::MvNorm(*f, norm_f);
    if (MyPID == 0)
      std::cout << "\nFinal residual norm = " << norm_f[0] << std::endl;

    // Print response
    std::cout << "\nResponse =      " << std::endl;
    //Writer::writeDense(std::cout, g);
    Writer::writeDenseFile("stochastic_residual.mm", f);















/*
    double g_mean = g->get1dView()[0].mean();
    double g_std_dev = g->get1dView()[0].standard_deviation();
    std::cout << "g mean = " << g_mean << std::endl;
    std::cout << "g std_dev = " << g_std_dev << std::endl;
    bool passed = false;
    if (norm_f[0] < 1.0e-10 &&
        std::abs(g_mean-g_mean_exp) < g_tol &&
        std::abs(g_std_dev - g_std_dev_exp) < g_tol)
      passed = true;
    if (MyPID == 0) {
      if (passed)
        std::cout << "Example Passed!" << std::endl;
      else{
        std::cout << "Example Failed!" << std::endl;
        std::cout << "expected g_mean = "<< g_mean_exp << std::endl;
        std::cout << "expected g_std_dev = "<< g_std_dev_exp << std::endl;
      }
    }
*/






    }



    Teuchos::TimeMonitor::summarize(std::cout);
    Teuchos::TimeMonitor::zeroOutTimers();

  }
  
  catch (std::exception& e) {
    std::cout << e.what() << std::endl;
  }
  catch (string& s) {
    std::cout << s << std::endl;
  }
  catch (char *s) {
    std::cout << s << std::endl;
  }
  catch (...) {
    std::cout << "Caught unknown exception!" <<std:: endl;
  }

#ifdef HAVE_MPI
  MPI_Finalize() ;
#endif

}
int main(int argc, char *argv[]) 
{
  using std::cout;
  using std::endl;

  int info = 0;
  bool boolret;
  int MyPID;

#ifdef HAVE_MPI
  // Initialize MPI
  MPI_Init(&argc,&argv);
  MPI_Comm_rank( MPI_COMM_WORLD, &MyPID );
#else
  MyPID = 0;
#endif

  bool testFailed;
  bool verbose = false;
  bool debug = false;
  bool skinny = true;
  std::string filename("mhd1280b.cua");
  std::string which("LR");

  CommandLineProcessor cmdp(false,true);
  cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
  cmdp.setOption("skinny","hefty",&skinny,"Use a skinny (low-mem) or hefty (higher-mem) implementation of IRTR.");
  cmdp.setOption("debug","nodebug",&debug,"Print debugging information.");
  cmdp.setOption("filename",&filename,"Filename for Harwell-Boeing test matrix.");
  cmdp.setOption("sort",&which,"Targetted eigenvalues (SR or LR).");
  if (cmdp.parse(argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
    MPI_Finalize();
#endif
    return -1;
  }

#ifndef HAVE_ANASAZI_TRIUTILS
  cout << "This test requires Triutils. Please configure with --enable-triutils." << endl;
#ifdef HAVE_MPI
  MPI_Finalize() ;
#endif
  if (verbose && MyPID == 0) {
    cout << "End Result: TEST FAILED" << endl;	
  }
  return -1;
#endif

#ifdef HAVE_COMPLEX
  typedef std::complex<double> ScalarType;
#elif HAVE_COMPLEX_H
  typedef ::complex<double> ScalarType;
#else
  typedef double ScalarType;
  // no complex. quit with failure.
  if (verbose && MyPID == 0) {
    cout << "Not compiled with complex support." << endl;
    cout << "End Result: TEST FAILED" << endl;
#ifdef HAVE_MPI
    MPI_Finalize();
#endif
    return -1;
  }
#endif
  typedef ScalarTraits<ScalarType>                   SCT;
  typedef SCT::magnitudeType               MagnitudeType;
  typedef Anasazi::MultiVec<ScalarType>               MV;
  typedef Anasazi::Operator<ScalarType>               OP;
  typedef Anasazi::MultiVecTraits<ScalarType,MV>     MVT;
  typedef Anasazi::OperatorTraits<ScalarType,MV,OP>  OPT;
  const ScalarType ONE  = SCT::one();

  if (verbose && MyPID == 0) {
    cout << Anasazi::Anasazi_Version() << endl << endl;
  }

  //  Problem information
  int dim,dim2,nnz;
  double *dvals;
  int *colptr,*rowind;
  nnz = -1;
  info = readHB_newmat_double(filename.c_str(),&dim,&dim2,&nnz,
                              &colptr,&rowind,&dvals);
  if (info == 0 || nnz < 0) {
    if (verbose && MyPID == 0) {
      cout << "Error reading '" << filename << "'" << endl
           << "End Result: TEST FAILED" << endl;
    }
#ifdef HAVE_MPI
    MPI_Finalize();
#endif
    return -1;
  }
  // Convert interleaved doubles to complex values
  std::vector<ScalarType> cvals(nnz);
  for (int ii=0; ii<nnz; ii++) {
    cvals[ii] = ScalarType(dvals[ii*2],dvals[ii*2+1]);
  }
  // Build the problem matrix
  RCP< const MyBetterOperator<ScalarType> > K 
    = rcp( new MyBetterOperator<ScalarType>(dim,colptr,nnz,rowind,&cvals[0]) );

  // Create initial vectors
  int blockSize = 5;
  RCP<MyMultiVec<ScalarType> > ivec = rcp( new MyMultiVec<ScalarType>(dim,blockSize) );
  ivec->MvRandom();

  // Create eigenproblem
  const int nev = 4;
  RCP<Anasazi::BasicEigenproblem<ScalarType,MV,OP> > problem =
    rcp( new Anasazi::BasicEigenproblem<ScalarType,MV,OP>(K,ivec) );
  //
  // Inform the eigenproblem that the operator K is symmetric
  problem->setHermitian(true);
  //
  // Set the number of eigenvalues requested
  problem->setNEV( nev );
  //
  // Inform the eigenproblem that you are done passing it information
  boolret = problem->setProblem();
  if (boolret != true) {
    if (verbose && MyPID == 0) {
      cout << "Anasazi::BasicEigenproblem::SetProblem() returned with error." << endl
           << "End Result: TEST FAILED" << endl;
    }
#ifdef HAVE_MPI
    MPI_Finalize() ;
#endif
    return -1;
  }

  // Set verbosity level
  int verbosity = Anasazi::Errors + Anasazi::Warnings;
  if (verbose) {
    verbosity += Anasazi::IterationDetails + Anasazi::FinalSummary + Anasazi::TimingDetails;
  }
  if (debug) {
    verbosity += Anasazi::Debug;
  }


  // Eigensolver parameters
  int maxIters = 450;
  MagnitudeType tol = 1.0e-6;
  //
  // Create parameter list to pass into the solver manager
  ParameterList MyPL;
  MyPL.set( "Skinny Solver", skinny);
  MyPL.set( "Verbosity", verbosity );
  MyPL.set( "Which", which );
  MyPL.set( "Block Size", blockSize );
  MyPL.set( "Maximum Iterations", maxIters );
  MyPL.set( "Convergence Tolerance", tol );
  //
  // Create the solver manager
  Anasazi::RTRSolMgr<ScalarType,MV,OP> MySolverMan(problem, MyPL);

  // Solve the problem to the specified tolerances or length
  Anasazi::ReturnType returnCode = MySolverMan.solve();
  testFailed = false;
  if (returnCode != Anasazi::Converged) {
    testFailed = true;
  }

  // Get the eigenvalues and eigenvectors from the eigenproblem
  Anasazi::Eigensolution<ScalarType,MV> sol = problem->getSolution();
  RCP<MV> evecs = sol.Evecs;
  int numev = sol.numVecs;

  if (numev > 0) {

    std::ostringstream os;
    os.setf(std::ios::scientific, std::ios::floatfield);
    os.precision(6);

    // Compute the direct residual
    std::vector<MagnitudeType> normV( numev );
    SerialDenseMatrix<int,ScalarType> T(numev,numev);
    for (int i=0; i<numev; i++) {
      T(i,i) = sol.Evals[i].realpart;
    }
    RCP<MV> Kvecs = MVT::Clone( *evecs, numev );

    OPT::Apply( *K, *evecs, *Kvecs );

    MVT::MvTimesMatAddMv( -ONE, *evecs, T, ONE, *Kvecs );
    MVT::MvNorm( *Kvecs, normV );
  
    os << "Direct residual norms computed in IRTRComplex_test.exe" << endl
       << std::setw(20) << "Eigenvalue" << std::setw(20) << "Residual(M)" << endl
       << "----------------------------------------" << endl;
    for (int i=0; i<numev; i++) {
      if ( SCT::magnitude(sol.Evals[i].realpart) != SCT::zero() ) {
        normV[i] = SCT::magnitude(normV[i]/sol.Evals[i].realpart);
      }
      os << std::setw(20) << sol.Evals[i].realpart << std::setw(20) << normV[i] << endl;
      if ( normV[i] > tol ) {
        testFailed = true;
      }
    }
    if (verbose && MyPID==0) {
      cout << endl << os.str() << endl;
    }
  }

#ifdef HAVE_MPI
  MPI_Finalize() ;
#endif

  // Clean up.
  std::free( dvals );
  std::free( colptr );
  std::free( rowind );

  if (testFailed) {
    if (verbose && MyPID==0) {
      cout << "End Result: TEST FAILED" << endl;
    }
    return -1;
  }
  //
  // Default return value
  //
  if (verbose && MyPID==0) {
    cout << "End Result: TEST PASSED" << endl;
  }
  return 0;

}
Exemple #26
0
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
  MPI_Init(&argc,&argv);
#endif

  bool verbose = false;
  bool success = false;
  try {
    bool debug = false;

    // number of global elements
    const int blockSize = 5;
    const int dim = 99;

    CommandLineProcessor cmdp(false,true);
    cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
    cmdp.setOption("debug","nodebug",&debug,"Print debugging info.");
    if (cmdp.parse(argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
      MPI_Finalize();
#endif
      return -1;
    }

    //
    // Issue several useful typedefs;
    typedef double                    ST;
    typedef MultiVec<ST>              MV;
    typedef Operator<ST>              OP;
    typedef ScalarTraits<ST>         SCT;
    typedef SCT::magnitudeType        MT;

    //
    // Create an output manager
    RCP<OutputManager<ST> > printer
      = rcp( new BasicOutputManager<ST>() );
    int verbosity = Errors;
    if (verbose || debug) {
      verbosity += Warnings;
    }
    if (debug) {
      verbosity += Debug;
    }
    printer->setVerbosity( verbosity );
    //
    // Create a sort manager
    RCP< SortManager<MT> > sorter =
      rcp( new BasicSort<MT>("LM") );
    //
    // Create an orthogonalization manager
    RCP< MatOrthoManager<ST,MV,OP> > ortho =
      rcp( new SVQBOrthoManager<ST,MV,OP>() );

    printer->stream(Warnings) << Anasazi_Version() << std::endl << std::endl;

    //
    // Create an identity matrix
    std::vector<ST> diag(dim);
    for (int i=0; i<dim; i++) diag[i] = 1.0;
    RCP<MyOperator<ST> > I = rcp( new MyOperator<ST>(diag) );
    //
    // Create the solution eigenvectors
    std::vector<SCT::magnitudeType> v(blockSize);
    RCP< MyMultiVec<ST> > ivec = rcp( new MyMultiVec<ST>(dim,blockSize) );
    for (int i=0; i<blockSize; i++) (*ivec)(i,i) = 1.0;
    //
    // Create the solution eigenvalues
    //
    std::vector<SCT::magnitudeType> T(blockSize);
    for (int i=0; i<blockSize; i++) T[i] = 1.0;
    //
    // Create the residual vectors
    RCP< MyMultiVec<ST> > R = rcp( new MyMultiVec<ST>(dim,blockSize) );
    //
    // Create an eigenvalue problem
    RCP< Eigenproblem<ST,MV,OP> > problem =
      rcp( new BasicEigenproblem<ST,MV,OP>(I,ivec) );
    problem->setHermitian(true);
    problem->setNEV(blockSize);
    problem->setProblem();

    //
    // Create a StatusTestCombo (this will wrap around all tests)
    StatusTestCombo<ST,MV,OP> stcombo;
    //
    // Create a StatusTestOutput (this will also wrap around all tests)
    StatusTestOutput<ST,MV,OP> stoutput(printer,null,1,Passed+Failed+Undefined);
    //
    // Create a StatusTestMaxIters
    StatusTestMaxIters<ST,MV,OP> stmaxiter(1);
    //
    // Create a StatusTestResNorm
    StatusTestResNorm<ST,MV,OP> stresnorm(SCT::zero());

    //
    // Create a parameter list
    ParameterList pls;
    pls.set("Block Size",blockSize);
    //
    // Create an eigensolver for testing
    LOBPCG<ST,MV,OP> lobpcg(problem,sorter,printer,rcp(&stoutput,false),ortho,pls);
    //
    // Initialize the solver with the exact eigensolution
    {
      LOBPCGState<ST,MV> state;
      state.X = ivec;
      state.R = R;
      state.T = rcp( &T, false );
      lobpcg.initialize(state);
    }


    ////////////////////////////////////////////////////////////////////////////////
    //
    // Perform tests
    //
    try {

      //
      // test StatusTestOutput
      {
        //
        // stoutput currently has null child pointer
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getChild() != null, get_out, "StatusTestOutput::getChild() should have returned Teuchos::null.");
        //
        // calling checkStatus() with null child pointer should result in a StatusTestError exception
        bool threw_expected_exception;
        try {
          stoutput.checkStatus(&lobpcg);
          threw_expected_exception = false;
        }
        catch (const StatusTestError &ste) {
          threw_expected_exception = true;
        }
        TEUCHOS_TEST_FOR_EXCEPTION( threw_expected_exception == false, get_out, "StatusTestOutput::checkStatus() should have thrown exception.");
      }

      //
      // test StatusTestResNorm
      {
        stoutput.setChild(rcp(&stresnorm,false));
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Undefined, get_out, "StatusTestOutput::setChild() should reset status to Undefined.");
        //
        // solver has residual norms == 0 < SCT::prec()
        printer->print(Warnings,"*** StatusTestResNorm: 0 < -prec: Failed.\n");
        stresnorm.setTolerance(-SCT::prec());  // 0 < -prec() == false
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestResNorm::setTolerance() should reset status to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg) != Failed, get_out, "StatusTestResNorm::checkStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Failed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()          != Failed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.howMany() != 0, get_out, "StatusTestResNorm::howMany() should have returned 0.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.whichVecs().size() != 0, get_out, "StatusTestResNorm::whichVecs() should have been empty.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()  != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");

        printer->print(Warnings,"*** StatusTestResNorm: 0 < prec: Passed.\n");
        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestResNorm::setTolerance() should reset status to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg) != Passed, get_out, "StatusTestResNorm::checkStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()          != Passed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.howMany() != blockSize, get_out, "StatusTestResNorm::howMany() should have returned blockSize.");
        TEUCHOS_TEST_FOR_EXCEPTION( (int)stresnorm.whichVecs().size() != blockSize, get_out, "StatusTestResNorm::whichVecs() should have had length blockSize.");
        std::vector<int> whch(stresnorm.whichVecs());
        for (int i=0; i<(int)whch.size(); i++) {
          TEUCHOS_TEST_FOR_EXCEPTION( whch[i] != i, get_out, "StatusTestResNorm::howMany() should have contained {0,blockSize-1}.");
        }
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()  != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
      }

      //
      // test StatusTestMaxIters
      {
        stoutput.setChild(rcp(&stmaxiter,false));
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Undefined, get_out, "StatusTestOutput::setChild() should reset status to Undefined.");
        //
        // solver has numIters() == 0
        printer->print(Warnings,"*** StatusTestMaxIters: 0 >= 1: Failed.\n");
        stmaxiter.setMaxIters(1); // 0 >= 1 == false
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::setMaxIters() should reset status to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg) != Failed, get_out, "StatusTestMaxIters::checkStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Failed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()          != Failed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()  != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::clearStatus() should reset status to Undefined.");

        printer->print(Warnings,"*** StatusTestMaxIters: 0 >= 0: Passed.\n");
        stmaxiter.setMaxIters(0); // 0 >= 0 == true
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::setMaxIters() should reset status to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg) != Passed, get_out, "StatusTestMaxIters::checkStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Passed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()          != Passed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()  != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::clearStatus() should reset status to Undefined.");

        printer->print(Warnings,"*** StatusTestMaxIters: 0 < 0: Failed.\n");
        stmaxiter.setNegate(true); // 0 < 0 == false
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::setMaxIters() should reset status to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg) != Failed, get_out, "StatusTestMaxIters::checkStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Failed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()          != Failed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()  != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::clearStatus() should reset status to Undefined.");

        printer->print(Warnings,"*** StatusTestMaxIters: 0 < 1: Passed.\n");
        stmaxiter.setMaxIters(1); // 0 < 1 == true
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::setMaxIters() should reset status to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg) != Passed, get_out, "StatusTestMaxIters::checkStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Passed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()          != Passed, get_out, "StatusTestResNorm::getStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()  != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestMaxIters::clearStatus() should reset status to Undefined.");
      }

      //
      // test StatusTestCombo(AND)
      //
      // also test clearStatus()
      {
        stoutput.setChild(rcp(&stcombo,false));
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.getStatus()          != Undefined, get_out, "StatusTestOutput::setChild() should reset status to Undefined.");
        stcombo.setTests( tuple<RCP<StatusTest<ST,MV,OP> > >(rcp(&stresnorm,false),rcp(&stmaxiter,false)) );
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getTests().size() != 2, get_out, "StatusTestCombo::getTests() should have two tests.");
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getStatus()    != Undefined, get_out, "StatusTestCombo::setTests() should reset status to Undefined.");
        stcombo.setComboType( stcombo.AND );
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getComboType() != stcombo.AND, get_out, "StatusTestCombo::getComboType() should be AND.");
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getStatus()    != Undefined, get_out, "StatusTestCombo::setComboType() should reset status to Undefined.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(false);
        stmaxiter.setMaxIters(0);             // 0 >= 0     == true (second test)
        // test that T & T => T
        printer->print(Warnings,"*** StatusTestCombo(AND): T & T: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Passed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F & T => F
        printer->print(Warnings,"*** StatusTestCombo(AND): F & T: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Passed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(true);  // 0 < 0 == false   (second test)
        // test that T & F => F
        printer->print(Warnings,"*** StatusTestCombo(AND): T & F: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed , get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed , get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed , get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed , get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Failed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F & F => F
        printer->print(Warnings,"*** StatusTestCombo(AND): F & F: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed , get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed , get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed , get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed , get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Failed , get_out, "StatusTestMaxIters::clearStatus() unexpected return.");
        stoutput.clearStatus();
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus() != Undefined, get_out, "StatusTestOutput::clearStatus() should reset all to Undefined.");
      }

      //
      // test StatusTestCombo(OR)
      {
        stcombo.setComboType( stcombo.OR );
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getComboType() != stcombo.OR, get_out, "StatusTestCombo::getComboType() should be OR.");
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getStatus()    != Undefined, get_out, "StatusTestCombo::setComboType() should reset status to Undefined.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(false);
        stmaxiter.setMaxIters(0);             // 0 >= 0     == true (second test)
        // test that T | T => T
        printer->print(Warnings,"*** StatusTestCombo(OR): T | T: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Passed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F | T => T
        printer->print(Warnings,"*** StatusTestCombo(OR): F | T: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Passed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(true);  // 0 < 0 == false   (second test)
        // test that T | F => T
        printer->print(Warnings,"*** StatusTestCombo(OR): T | F: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Failed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F | F => F
        printer->print(Warnings,"*** StatusTestCombo(OR): F | F: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Failed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");
      }

      //
      // test StatusTestCombo(SEQAND)
      {
        stcombo.setComboType( stcombo.SEQAND );
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getComboType() != stcombo.SEQAND, get_out, "StatusTestCombo::getComboType() should be SEQAND.");
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getStatus()    != Undefined, get_out, "StatusTestCombo::setComboType() should reset status to Undefined.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(false);
        stmaxiter.setMaxIters(0);             // 0 >= 0     == true (second test)
        // test that T && T => T
        printer->print(Warnings,"*** StatusTestCombo(SEQAND): T && T: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Passed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F && U => F
        printer->print(Warnings,"*** StatusTestCombo(SEQAND): F && U: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Undefined, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(true);  // 0 < 0 == false   (second test)
        // test that T && F => F
        printer->print(Warnings,"*** StatusTestCombo(SEQAND): T && F: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Failed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F && U => F
        printer->print(Warnings,"*** StatusTestCombo(SEQAND): F && U: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Undefined, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");
      }

      //
      // test StatusTestCombo(SEQOR)
      {
        stcombo.setComboType( stcombo.SEQOR );
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getComboType() != stcombo.SEQOR, get_out, "StatusTestCombo::getComboType() should be SEQOR.");
        TEUCHOS_TEST_FOR_EXCEPTION( stcombo.getStatus()    != Undefined, get_out, "StatusTestCombo::setComboType() should reset status to Undefined.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(false);
        stmaxiter.setMaxIters(0);             // 0 >= 0     == true (second test)
        // test that T || U => T
        printer->print(Warnings,"*** StatusTestCombo(SEQOR): T || U: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Undefined, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F || T => T
        printer->print(Warnings,"*** StatusTestCombo(SEQOR): F || T: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Passed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(SCT::prec());  // 0 < prec() == true (first test)
        stmaxiter.setNegate(true);  // 0 < 0 == false   (second test)
        // test that T || U => T
        printer->print(Warnings,"*** StatusTestCombo(SEQOR): T || U: Passed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Passed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Passed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Passed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Passed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Undefined, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");

        stresnorm.setTolerance(-SCT::prec()); // 0 < -prec() == false (first test)
        // test that F || F => F
        printer->print(Warnings,"*** StatusTestCombo(SEQOR): F || F: Failed.\n");
        TEUCHOS_TEST_FOR_EXCEPTION( stoutput.checkStatus(&lobpcg)   != Failed, get_out, "StatusTestOutput::checkStatus(): unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(  stoutput.getStatus()           != Failed, get_out, "StatusTestOutput::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION(   stcombo.getStatus()           != Failed, get_out, "StatusTestCombo::getStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stresnorm.getStatus()           != Failed, get_out, "StatusTestResNorm::clearStatus() unexpected return.");
        TEUCHOS_TEST_FOR_EXCEPTION( stmaxiter.getStatus()           != Failed, get_out, "StatusTestMaxIters::clearStatus() unexpected return.");
      }
      success = true;
    } // end of try
    catch (const get_out &go) {
      printer->stream(Warnings) << go.what() << std::endl;
    }

    printer->print(Warnings,"\n");

    if (success)
      printer->print(Warnings,"End Result: TEST PASSED\n");
    else
      printer->print(Warnings,"End Result: TEST FAILED\n");
  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);

#ifdef HAVE_MPI
  MPI_Finalize();
#endif

  return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main(int argc, char *argv[]) {
  //
  int MyPID = 0;
#ifdef EPETRA_MPI
  // Initialize MPI
  MPI_Init(&argc,&argv);
  Epetra_MpiComm Comm(MPI_COMM_WORLD);
  MyPID = Comm.MyPID();
#else
  Epetra_SerialComm Comm;
#endif
  //
  typedef double                            ST;
  typedef Teuchos::ScalarTraits<ST>        SCT;
  typedef SCT::magnitudeType                MT;
  typedef Epetra_MultiVector                MV;
  typedef Epetra_Operator                   OP;
  typedef Belos::MultiVecTraits<ST,MV>     MVT;
  typedef Belos::OperatorTraits<ST,MV,OP>  OPT;

  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;

  bool success = false;
  bool verbose = false;

  try {
    bool debug = false, proc_verbose = false;
    int frequency = -1;        // frequency of status test output.
    int numrhs = 1;            // number of right-hand sides to solve for
    int maxiters = -1;         // maximum number of iterations allowed per linear system
    MT tol = 1.0e-10;           // relative residual tolerance

    Teuchos::CommandLineProcessor cmdp(false,true);
    cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
    cmdp.setOption("debug","nodebug",&debug,"Print debugging information from the solver.");
    cmdp.setOption("frequency",&frequency,"Solvers frequency for printing residuals (#iters).");
    cmdp.setOption("tol",&tol,"Relative residual tolerance used by GMRES solver.");
    cmdp.setOption("num-rhs",&numrhs,"Number of right-hand sides to be solved for.");
    cmdp.setOption("max-iters",&maxiters,"Maximum number of iterations per linear system (-1 = adapted to problem/block size).");
    if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
      return -1;
    }
    if (!verbose)
      frequency = -1;  // reset frequency if test is not verbose
    // **********************************************************************
    // ******************Set up the problem to be solved*********************
    // construct diagonal matrix
    const int NumGlobalElements = 100;
    const int m = 4; // number of negative eigenvalues

    // Create diagonal matrix with n-m positive and m negative eigenvalues.
    Epetra_Map epetraMap( NumGlobalElements, 0, Comm );

    Teuchos::RCP<Epetra_CrsMatrix> A = Teuchos::rcp( new Epetra_CrsMatrix( Copy, epetraMap, 1 ) );
    for ( int k=0; k<epetraMap.NumMyElements(); k++ )
    {
      int GIDk = epetraMap.GID(k);
      double val = 2*(GIDk-m) + 1;
      TEUCHOS_ASSERT_EQUALITY( 0, A->InsertGlobalValues( GIDk, 1, &val, &GIDk ) );
    }
    TEUCHOS_ASSERT_EQUALITY( 0, A->FillComplete() );
    TEUCHOS_ASSERT_EQUALITY( 0, A->OptimizeStorage() );

    // create initial guess and right-hand side
    Teuchos::RCP<Epetra_MultiVector> vecX = Teuchos::rcp( new Epetra_MultiVector( epetraMap, numrhs ) );

    Teuchos::RCP<Epetra_MultiVector> vecB = Teuchos::rcp( new Epetra_MultiVector( epetraMap, numrhs ) );
    // **********************************************************************
    proc_verbose = verbose && (MyPID==0);  /* Only print on the zero processor */


    Teuchos::RCP<Epetra_MultiVector> X;
    Teuchos::RCP<Epetra_MultiVector> B;
    // Check to see if the number of right-hand sides is the same as requested.
    if (numrhs>1) {
      X = rcp( new Epetra_MultiVector( epetraMap, numrhs ) );
      B = rcp( new Epetra_MultiVector( epetraMap, numrhs ) );
      X->Random();
      OPT::Apply( *A, *X, *B );
      X->PutScalar( 0.0 );
    }
    else {
      X = Teuchos::rcp_implicit_cast<Epetra_MultiVector>(vecX);
      B = Teuchos::rcp_implicit_cast<Epetra_MultiVector>(vecB);
      B->PutScalar( 1.0 );
    }
    //
    // ********Other information used by block solver***********
    // *****************(can be user specified)******************
    //
    if (maxiters == -1)
      maxiters = NumGlobalElements - 1; // maximum number of iterations to run
    //
    ParameterList belosList;
    belosList.set( "Maximum Iterations", maxiters );        // Maximum number of iterations allowed
    belosList.set( "Convergence Tolerance", tol );          // Relative convergence tolerance requested
    belosList.set( "Assert Positive Definiteness", false ); // Explicitly don't enforce positive definiteness

    int verbosity = Belos::Errors + Belos::Warnings;
    if (verbose) {
      verbosity += Belos::TimingDetails + Belos::StatusTestDetails;
      if (frequency > 0)
        belosList.set( "Output Frequency", frequency );
    }
    if (debug) {
      verbosity += Belos::Debug;
    }
    belosList.set( "Verbosity", verbosity );
    //
    // Construct an unpreconditioned linear problem instance.
    //
    Belos::LinearProblem<double,MV,OP> problem( A, X, B );
    bool set = problem.setProblem();
    if (set == false) {
      if (proc_verbose)
        std::cout << std::endl << "ERROR:  Belos::LinearProblem failed to set up correctly!" << std::endl;
      return -1;
    }
    //
    // *******************************************************************
    // ****************Start the CG iteration*************************
    // *******************************************************************
    //
    // Create an iterative solver manager.
    RCP<Belos::SolverManager<double,MV,OP> > newSolver
      = rcp( new Belos::PseudoBlockCGSolMgr<double,MV,OP>(rcp(&problem,false), rcp(&belosList,false)));

    //
    // **********Print out information about problem*******************
    //
    if (proc_verbose) {
      std::cout << std::endl << std::endl;
      std::cout << "Dimension of matrix: " << NumGlobalElements << std::endl;
      std::cout << "Number of right-hand sides: " << numrhs << std::endl;
      std::cout << "Relative residual tolerance: " << tol << std::endl;
      std::cout << std::endl;
    }
    //
    // Perform solve
    //
    Belos::ReturnType ret = newSolver->solve();
    //
    // Get the number of iterations for this solve.
    //
    int numIters = newSolver->getNumIters();
    if (proc_verbose)
      std::cout << "Number of iterations performed for this solve: " << numIters << std::endl;
    //
    // Compute actual residuals.
    //
    bool badRes = false;
    std::vector<double> actual_resids( numrhs );
    std::vector<double> rhs_norm( numrhs );
    Epetra_MultiVector resid(epetraMap, numrhs);
    OPT::Apply( *A, *X, resid );
    MVT::MvAddMv( -1.0, resid, 1.0, *B, resid );
    MVT::MvNorm( resid, actual_resids );
    MVT::MvNorm( *B, rhs_norm );
    if (proc_verbose) {
      std::cout<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
      for ( int i=0; i<numrhs; i++) {
        double actRes = actual_resids[i]/rhs_norm[i];
        std::cout<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
        if (actRes > tol) badRes = true;
      }
    }

    success = ret==Belos::Converged && !badRes;

    if (success) {
      if (proc_verbose)
        std::cout << "End Result: TEST PASSED" << std::endl;
    } else {
      if (proc_verbose)
        std::cout << "End Result: TEST FAILED" << std::endl;
    }
  }
  TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);

#ifdef EPETRA_MPI
  MPI_Finalize();
#endif

  return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
int main(int argc, char *argv[]) 
{
#ifdef HAVE_MPI  
  MPI_Init(&argc, &argv);
#endif

  using namespace std;
  using namespace Teuchos;
  using namespace PHX;
  
  try {
    
    RCP<Time> total_time = TimeMonitor::getNewTimer("Total Run Time");
    TimeMonitor tm(*total_time);

    RCP<Time> residual_eval_time = 
      TimeMonitor::getNewTimer("Residual Evaluation Time");
    RCP<Time> jacobian_eval_time = 
      TimeMonitor::getNewTimer("Jacobian Evaluation Time");
    RCP<Time> linear_solve_time = 
      TimeMonitor::getNewTimer("Linear Solve Time");
    RCP<Time> nonlinear_solve_time = 
      TimeMonitor::getNewTimer("Nonlinear Solve Time");
    RCP<Time> preconditioner_solve_time = 
      TimeMonitor::getNewTimer("Preconditioner Time");
    RCP<Time> setup_time = 
      TimeMonitor::getNewTimer("Setup Time (not scalable)");
    RCP<Time> jv_eval_time = 
      TimeMonitor::getNewTimer("Jv (AD)");
    RCP<Time> matvec = 
      TimeMonitor::getNewTimer("Matvec");

    setup_time->start();

    bool print_debug_info = false;

#ifdef HAVE_MPI
    RCP<Epetra_Comm> comm = rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
#else
    RCP<Epetra_Comm> comm = rcp(new Epetra_SerialComm);
#endif
    
    Teuchos::basic_FancyOStream<char> os(rcp(&std::cout,false));
    os.setShowProcRank(true);
    os.setProcRankAndSize(comm->MyPID(), comm->NumProc());

    if (comm->MyPID() == 0)
      cout << "\nStarting FEM_Nonlinear Example!\n" << endl;

    // *********************************************************
    // * Build the Finite Element data structures
    // *********************************************************

    // Problem dimension - a 2D problem
    const static int dim = 2;

    // Create the mesh
    MeshBuilder mb(comm, 10, 3, 1.0, 1.0, 8);

    if (print_debug_info) 
      os << mb;

    std::vector<Element_Linear2D>& cells = *(mb.myElements());

    // Divide mesh into workset blocks
    const std::size_t workset_size = 15;
    std::vector<MyWorkset> worksets;
    {
      std::vector<Element_Linear2D>::iterator cell_it = cells.begin();
      std::size_t count = 0;
      MyWorkset w;
      w.local_offset = cell_it->localElementIndex();
      w.begin = cell_it;
      for (; cell_it != cells.end(); ++cell_it) {
	++count;
	std::vector<Element_Linear2D>::iterator next = cell_it;
	++next;
	
	if ( count == workset_size || next == cells.end()) {
	  w.end = next;
	  w.num_cells = count;
	  worksets.push_back(w);
	  count = 0;
	  
	  if (next != cells.end()) {
	    w.local_offset = next->localElementIndex();
	    w.begin = next;
	  }
	}
      }
    }
    
    if (print_debug_info) {
      cout << "Printing Element Information" << endl;
      for (std::size_t i = 0; i < worksets.size(); ++i) {
	std::vector<Element_Linear2D>::iterator it = worksets[i].begin;
	for (; it != worksets[i].end; ++it)
	  cout << *it << endl;
      }
    }
    
    if (print_debug_info) {
      for (std::size_t i = 0; i < worksets.size(); ++i) {
	cout << "Printing Workset Information" << endl;
	cout << "worksets[" << i << "]" << endl;
	cout << "  num_cells =" << worksets[i].num_cells << endl;
	cout << "  local_offset =" << worksets[i].local_offset << endl;
	std::vector<Element_Linear2D>::iterator it = worksets[i].begin;
	for (; it != worksets[i].end; ++it)
	  cout << "  cell_local_index =" << it->localElementIndex() << endl;
      }
      cout << endl;
    }

    // *********************************************************
    // * Build the Newton solver data structures
    // *********************************************************

    // Setup Nonlinear Problem (build Epetra_Vector and Epetra_CrsMatrix)
    // Newton's method: J delta_x = -f
    const std::size_t num_eq = 2;
    
    LinearObjectFactory lof(mb, comm, num_eq);

    if (print_debug_info) {
      ofstream file("OwnedGraph.dat", ios::out | ios::app);
      Teuchos::basic_FancyOStream<char> p(rcp(&file,false)); 
      p.setShowProcRank(true); 
      p.setProcRankAndSize(comm->MyPID(), comm->NumProc()); 	
      lof.ownedGraph()->Print(p);
    }

    Epetra_Map owned_map = *(lof.ownedMap());
    Epetra_Map overlapped_map = *(lof.overlappedMap());
    Epetra_CrsGraph owned_graph = *(lof.ownedGraph());
    Epetra_CrsGraph overlapped_graph = *(lof.overlappedGraph());

    // Solution vector x
    RCP<Epetra_Vector> owned_x = rcp(new Epetra_Vector(owned_map,true));
    RCP<Epetra_Vector> overlapped_x =  
      rcp(new Epetra_Vector(overlapped_map,true));

    // Update vector x
    RCP<Epetra_Vector> owned_delta_x = rcp(new Epetra_Vector(owned_map,true));

    // Residual vector f
    RCP<Epetra_Vector> owned_f = rcp(new Epetra_Vector(owned_map,true));
    RCP<Epetra_Vector> overlapped_f =  
      rcp(new Epetra_Vector(overlapped_map,true));

    // Jacobian Matrix
    Epetra_DataAccess copy = ::Copy;
    RCP<Epetra_CrsMatrix> owned_jac = 
      rcp(new Epetra_CrsMatrix(copy, owned_graph));
    RCP<Epetra_CrsMatrix> overlapped_jac = 
      rcp(new Epetra_CrsMatrix(copy, overlapped_graph));

    // Import/export
    RCP<Epetra_Import> importer = 
      rcp(new Epetra_Import(overlapped_map, owned_map));
    RCP<Epetra_Export> exporter = 
      rcp(new Epetra_Export(overlapped_map, owned_map));

    // Sets bc for initial guess
    applyBoundaryConditions(1.0, *owned_x, *owned_jac, *owned_f, mb);
    
    // *********************************************************
    // * Build the FieldManager
    // *********************************************************

    RCP< vector<string> > dof_names = rcp(new vector<string>(num_eq));
    (*dof_names)[0] = "Temperature";
    (*dof_names)[1] = "Velocity X";

    RCP<DataLayout> qp_scalar = 
      rcp(new MDALayout<Cell,QuadPoint>(workset_size,4));
    RCP<DataLayout> node_scalar = 
      rcp(new MDALayout<Cell,Node>(workset_size,4));
    
    RCP<DataLayout> qp_vec = 
      rcp(new MDALayout<Cell,QuadPoint,Dim>(workset_size,4,dim));
    RCP<DataLayout> node_vec = 
      rcp(new MDALayout<Cell,Node,Dim>(workset_size,4,dim));

    RCP<DataLayout> dummy = rcp(new MDALayout<Cell>(0));
    
    map<string, RCP<ParameterList> > evaluators_to_build;
    
    { // Gather Solution
      RCP<ParameterList> p = rcp(new ParameterList);
      int type = MyFactoryTraits<MyTraits>::id_gather_solution;
      p->set<int>("Type", type);
      p->set< RCP< vector<string> > >("Solution Names", dof_names);
      p->set< RCP<Epetra_Vector> >("Solution Vector", overlapped_x);
      p->set< RCP<DataLayout> >("Data Layout", node_scalar);
      evaluators_to_build["Gather Solution"] = p;
    }
    
    { // FE Interpolation - Temperature
      RCP<ParameterList> p = rcp(new ParameterList);
      
      int type = MyFactoryTraits<MyTraits>::id_feinterpolation;
      p->set<int>("Type", type);
      
      p->set<string>("Node Variable Name", "Temperature");
      p->set<string>("QP Variable Name", "Temperature");
      p->set<string>("Gradient QP Variable Name", "Temperature Gradient");
      
      p->set< RCP<DataLayout> >("Node Data Layout", node_scalar);
      p->set< RCP<DataLayout> >("QP Scalar Data Layout", qp_scalar);
      p->set< RCP<DataLayout> >("QP Vector Data Layout", qp_vec);
      
      evaluators_to_build["FE Interpolation Temperature"] = p;
    }
     
    { // FE Interpolation - Velocity X
      RCP<ParameterList> p = rcp(new ParameterList);
      
      int type = MyFactoryTraits<MyTraits>::id_feinterpolation;
      p->set<int>("Type", type);
      
      p->set<string>("Node Variable Name", "Velocity X");
      p->set<string>("QP Variable Name", "Velocity X");
      p->set<string>("Gradient QP Variable Name", "Velocity X Gradient");
      
      p->set< RCP<DataLayout> >("Node Data Layout", node_scalar);
      p->set< RCP<DataLayout> >("QP Scalar Data Layout", qp_scalar);
      p->set< RCP<DataLayout> >("QP Vector Data Layout", qp_vec);
      
      evaluators_to_build["FE Interpolation Velocity X"] = p;
    }

    { // Evaluate residual
      RCP<ParameterList> p = rcp(new ParameterList);
      int type = MyFactoryTraits<MyTraits>::id_equations;
      p->set<int>("Type", type);
      p->set< RCP< vector<string> > >("Solution Names", dof_names);
      p->set< RCP<DataLayout> >("Node Data Layout", node_scalar);
      p->set< RCP<DataLayout> >("QP Data Layout", qp_scalar);
      p->set< RCP<DataLayout> >("Gradient QP Data Layout", qp_vec);
      evaluators_to_build["Equations"] = p;
    }
 
    { // Scatter Solution
      RCP<ParameterList> p = rcp(new ParameterList);
      int type = MyFactoryTraits<MyTraits>::id_scatter_residual;
      p->set<int>("Type", type);

      RCP< vector<string> > res_names = rcp(new vector<string>(num_eq));
      (*res_names)[0] = "Residual Temperature";
      (*res_names)[1] = "Residual Velocity X";

      p->set< RCP< vector<string> > >("Residual Names", res_names);
      p->set< RCP<Epetra_Vector> >("Residual Vector", overlapped_f);
      p->set< RCP<Epetra_CrsMatrix> >("Jacobian Matrix", overlapped_jac);
      p->set< RCP<DataLayout> >("Dummy Data Layout", dummy);
      p->set< RCP<DataLayout> >("Data Layout", node_scalar);
      evaluators_to_build["Scatter Residual"] = p;
    }
    
    // Build Field Evaluators for each evaluation type
    EvaluatorFactory<MyTraits,MyFactoryTraits<MyTraits> > factory;
    RCP< vector< RCP<Evaluator_TemplateManager<MyTraits> > > > 
      evaluators;
    evaluators = factory.buildEvaluators(evaluators_to_build);
    
    // Create a FieldManager
    FieldManager<MyTraits> fm;
      
    // Register all Evaluators 
    registerEvaluators(evaluators, fm);

    // Request quantities to assemble RESIDUAL PDE operators
    {
      typedef MyTraits::Residual::ScalarT ResScalarT;
      Tag<ResScalarT> res_tag("Scatter", dummy);
      fm.requireField<MyTraits::Residual>(res_tag);
      
      // Request quantities to assemble JACOBIAN PDE operators
      typedef MyTraits::Jacobian::ScalarT JacScalarT;
      Tag<JacScalarT> jac_tag("Scatter", dummy);
      fm.requireField<MyTraits::Jacobian>(jac_tag);

      // Request quantities to assemble Jv operators
      typedef MyTraits::Jv::ScalarT JvScalarT;
      Tag<JvScalarT> jv_tag("Scatter", dummy);
      fm.requireField<MyTraits::Jv>(jv_tag);
    }

    {
      RCP<Time> registration_time = 
	TimeMonitor::getNewTimer("Post Registration Setup Time");
      {
	TimeMonitor t(*registration_time);
	fm.postRegistrationSetupForType<MyTraits::Residual>(NULL);
	fm.postRegistrationSetupForType<MyTraits::Jacobian>(NULL);
	fm.postRegistrationSetupForType<MyTraits::Jv>(NULL);
      }
    }

    if (print_debug_info)
      cout << fm << endl;

    // *********************************************************
    // * Evaluate Jacobian and Residual (required for ML to 
    // * to be constructed properly
    // *********************************************************    
    {
      //TimeMonitor t(*jacobian_eval_time);

      overlapped_x->Import(*owned_x, *importer, Insert);
      
      owned_f->PutScalar(0.0);
      overlapped_f->PutScalar(0.0);
      owned_jac->PutScalar(0.0);
      overlapped_jac->PutScalar(0.0);
      
      for (std::size_t i = 0; i < worksets.size(); ++i)
	fm.evaluateFields<MyTraits::Jacobian>(worksets[i]);
      
      owned_f->Export(*overlapped_f, *exporter, Add);
      owned_jac->Export(*overlapped_jac, *exporter, Add);
      
      applyBoundaryConditions(1.0, *owned_x, *owned_jac, *owned_f, mb);
    }

    // *********************************************************
    // * Build Preconditioner (Ifpack or ML)
    // *********************************************************    
    bool use_ml = false;

    RCP<Belos::EpetraPrecOp> belosPrec;
    RCP<Ifpack_Preconditioner> ifpack_prec;
    RCP<ML_Epetra::MultiLevelPreconditioner> ml_prec;

    if (!use_ml) {
      Ifpack Factory;
      std::string PrecType = "ILU";
      int OverlapLevel = 0;
      ifpack_prec = 
	Teuchos::rcp( Factory.Create(PrecType,owned_jac.get(),OverlapLevel) );
      ParameterList ifpackList;
      ifpackList.set("fact: drop tolerance", 1e-9);
      ifpackList.set("fact: level-of-fill", 1);
      ifpackList.set("schwarz: combine mode", "Add");
      IFPACK_CHK_ERR(ifpack_prec->SetParameters(ifpackList));
      IFPACK_CHK_ERR(ifpack_prec->Initialize());
      belosPrec = rcp( new Belos::EpetraPrecOp( ifpack_prec ) );
    }
    else {
      ParameterList ml_params;
      ML_Epetra::SetDefaults("SA",ml_params);
      //ml_params.set("Base Method Defaults", "SA");
      ml_params.set("ML label", "Phalanx_Test");
      ml_params.set("ML output", 10);
      ml_params.set("print unused", 1);
      ml_params.set("max levels", 4);
      ml_params.set("PDE equations",2);
      ml_params.set("prec type","MGV");
      ml_params.set("increasing or decreasing","increasing");
      // ml_params.set("aggregation: nodes per aggregate",50);
      ml_params.set("aggregation: type","Uncoupled");
      ml_params.set("aggregation: damping factor", 0.0);
      ml_params.set("coarse: type","Amesos-KLU");
      //ml_params.set("coarse: type","IFPACK");
      ml_params.set("coarse: max size", 1000);
      //ml_params.set("smoother: type","IFPACK");
      ml_params.set("smoother: type","block Gauss-Seidel");
      ml_params.set("smoother: ifpack type","ILU");
      ml_params.set("smoother: ifpack overlap",1);
      ml_params.sublist("smoother: ifpack list").set("fact: level-of-fill",1);
      ml_params.sublist("smoother: ifpack list").set("schwarz: reordering type","rcm");
      ml_prec = rcp( new ML_Epetra::MultiLevelPreconditioner(*owned_jac, 
							     ml_params, 
							     true) );
      belosPrec = rcp( new Belos::EpetraPrecOp( ml_prec ) );
    }

    // *********************************************************
    // * Build linear solver (Belos)
    // *********************************************************
    
    // Linear solver parameters
    typedef double                            ST;
    typedef Teuchos::ScalarTraits<ST>        SCT;
    typedef SCT::magnitudeType                MT;
    typedef Epetra_MultiVector                MV;
    typedef Epetra_Operator                   OP;
    typedef Belos::MultiVecTraits<ST,MV>     MVT;
    typedef Belos::OperatorTraits<ST,MV,OP>  OPT;
    
    RCP<ParameterList> belosList = rcp(new ParameterList);
    belosList->set<int>("Num Blocks", 400);
    belosList->set<int>("Block Size", 1);
    belosList->set<int>("Maximum Iterations", 400);
    belosList->set<int>("Maximum Restarts", 0);
    belosList->set<MT>( "Convergence Tolerance", 1.0e-4);
    int verbosity = Belos::Errors + Belos::Warnings;
    if (false) {
      verbosity += Belos::TimingDetails + Belos::StatusTestDetails;
      belosList->set<int>( "Output Frequency", -1);
    }
    if (print_debug_info) {
      verbosity += Belos::Debug;
      belosList->set<int>( "Output Frequency", -1);
    }
    belosList->set( "Verbosity", verbosity );
    
    RCP<Epetra_MultiVector> F = 
      Teuchos::rcp_implicit_cast<Epetra_MultiVector>(owned_f);
    
    RCP<Epetra_MultiVector> DX = 
      Teuchos::rcp_implicit_cast<Epetra_MultiVector>(owned_delta_x);
    
    RCP<Belos::LinearProblem<double,MV,OP> > problem =
      rcp(new Belos::LinearProblem<double,MV,OP>(owned_jac, DX, F) );

    problem->setRightPrec( belosPrec );

    RCP< Belos::SolverManager<double,MV,OP> > gmres_solver = 
      rcp( new Belos::BlockGmresSolMgr<double,MV,OP>(problem, belosList) );
    
    setup_time->stop();










    // Timing for Mat-Vec using sacado
    RCP<Epetra_Vector> owned_v = rcp(new Epetra_Vector(owned_map,true));
    RCP<Epetra_Vector> overlapped_v = 
      rcp(new Epetra_Vector(overlapped_map,true));
    RCP<Epetra_Vector> owned_Jv = rcp(new Epetra_Vector(owned_map,true));
    RCP<Epetra_Vector> overlapped_Jv = 
      rcp(new Epetra_Vector(overlapped_map,true));

    owned_x->PutScalar(1.0);
    owned_v->PutScalar(1.0);

    int iter = 0;
    while (iter != 30) {

      overlapped_x->Import(*owned_x, *importer, Insert);

      overlapped_v->Import(*owned_v, *importer, Insert);

      owned_f->PutScalar(0.0);
      overlapped_f->PutScalar(0.0);
      owned_jac->PutScalar(0.0);
      overlapped_jac->PutScalar(0.0);
      owned_Jv->PutScalar(0.0);
      overlapped_Jv->PutScalar(0.0);

      for (std::size_t i = 0; i < worksets.size(); ++i) {
	worksets[i].v = overlapped_v;
	worksets[i].Jv = overlapped_Jv;
      }
      {
	TimeMonitor t(*residual_eval_time);
	for (std::size_t i = 0; i < worksets.size(); ++i)
	  fm.evaluateFields<MyTraits::Residual>(worksets[i]);
      }
      {
	TimeMonitor t(*jacobian_eval_time);
	for (std::size_t i = 0; i < worksets.size(); ++i)
	  fm.evaluateFields<MyTraits::Jacobian>(worksets[i]);
      }
      {
	TimeMonitor t(*jv_eval_time);
	for (std::size_t i = 0; i < worksets.size(); ++i)
	  fm.evaluateFields<MyTraits::Jv>(worksets[i]);
      }

      owned_f->Export(*overlapped_f, *exporter, Add);
      owned_jac->Export(*overlapped_jac, *exporter, Add);
      owned_Jv->Export(*overlapped_Jv, *exporter, Add);

      applyBoundaryConditions(1.0, *owned_x, *owned_jac, *owned_f, mb);

      {
	TimeMonitor t(*matvec);
	owned_jac->Apply(*owned_x, *owned_f);
      }

      ++iter;
    }

    //owned_jac->Print(std::cout);
    //overlapped_Jv->Print(std::cout);
    //owned_Jv->Print(std::cout);
    




    


    // NOTE: in the future we can set up the nonlinear solver below to
    // do Jacobian-Free Newton-Krylov solves to test the matvec


    /*


    // *********************************************************
    // * Solve the system
    // *********************************************************

    // Set initial guess
    owned_x->PutScalar(1.0);

    // Evaluate Residual
    {
      TimeMonitor t(*residual_eval_time);

      overlapped_x->Import(*owned_x, *importer, Insert);

      owned_f->PutScalar(0.0);
      overlapped_f->PutScalar(0.0);

      for (std::size_t i = 0; i < worksets.size(); ++i)
	fm.evaluateFields<MyTraits::Residual>(worksets[i]);
      
      owned_f->Export(*overlapped_f, *exporter, Add);

      applyBoundaryConditions(1.0, *owned_x, *owned_jac, *owned_f, mb);
    }
    
    if (print_debug_info) {
      printVector("x_owned", *owned_x, -1);
      printVector("f_owned", *owned_f, -1);
    }

    // Newton Loop
    bool converged = false;
    std::size_t num_newton_steps = 0;
    std::size_t num_gmres_iterations = 0;
    checkConvergence(comm->MyPID(), num_newton_steps, *owned_f, 
		     *owned_delta_x, converged);

    while (!converged && num_newton_steps < 20) {
      
      TimeMonitor t(*nonlinear_solve_time);

      // Evaluate Residual and Jacobian
      {
	TimeMonitor t(*jacobian_eval_time);

	overlapped_x->Import(*owned_x, *importer, Insert);

	owned_f->PutScalar(0.0);
	overlapped_f->PutScalar(0.0);
	owned_jac->PutScalar(0.0);
	overlapped_jac->PutScalar(0.0);
	
	for (std::size_t i = 0; i < worksets.size(); ++i)
	  fm.evaluateFields<MyTraits::Jacobian>(worksets[i]);

	owned_f->Export(*overlapped_f, *exporter, Add);
	owned_jac->Export(*overlapped_jac, *exporter, Add);

	applyBoundaryConditions(1.0, *owned_x, *owned_jac, *owned_f, mb);
      }

      if (print_debug_info) {
	printVector("x_owned", *owned_x, num_newton_steps);
	printVector("x_overlapped", *overlapped_x, num_newton_steps);
	printVector("f_owned", *owned_f, num_newton_steps);
	printMatrix("jacobian_owned", *owned_jac, num_newton_steps);
      }
      
      owned_f->Scale(-1.0);

      // Solve linear problem
      {
	TimeMonitor t(*linear_solve_time);
	
	owned_delta_x->PutScalar(0.0);

	{
	  TimeMonitor tp(*preconditioner_solve_time);
	  
	  if (use_ml)
	    ml_prec->ReComputePreconditioner();
	  else
	    IFPACK_CHK_ERR(ifpack_prec->Compute());
	}

	problem->setProblem();

	Belos::ReturnType ret = gmres_solver->solve();

	int num_iters = gmres_solver->getNumIters();
	num_gmres_iterations += num_iters; 
	//if (print_debug_info) 
	if (comm->MyPID() == 0)
	  std::cout << "Number of gmres iterations performed for this solve: " 
		    << num_iters << std::endl;
	
	if (ret!=Belos::Converged && comm->MyPID() == 0) {
	  std::cout << std::endl << "WARNING:  Belos did not converge!" 
		    << std::endl;
	}
	
      }
      
      owned_x->Update(1.0, *owned_delta_x, 1.0);

      { // Evaluate Residual Only
	TimeMonitor t(*residual_eval_time);
	
	overlapped_x->Import(*owned_x, *importer, Insert);

	owned_f->PutScalar(0.0);
	overlapped_f->PutScalar(0.0);

	for (std::size_t i = 0; i < worksets.size(); ++i)
	  fm.evaluateFields<MyTraits::Residual>(worksets[i]);
	
	owned_f->Export(*overlapped_f, *exporter, Add);

	applyBoundaryConditions(1.0, *owned_x, *owned_jac, *owned_f, mb);
      }

      num_newton_steps += 1;

      checkConvergence(comm->MyPID(), num_newton_steps, *owned_f, 
		       *owned_delta_x, converged);
    }
     
    if (print_debug_info)
      printVector("f_owned", *owned_f, num_newton_steps);

    if (comm->MyPID() == 0) {
      if (converged)
	cout << "\n\nNewton Iteration Converged!\n" << endl;
      else
	cout << "\n\nNewton Iteration Failed to Converge!\n" << endl;
    }

    RCP<Time> file_io = 
      TimeMonitor::getNewTimer("File IO");
    {
      TimeMonitor t(*file_io);
      
      // Create a list of node coordinates
      std::map<int, std::vector<double> > coordinates;
      Teuchos::RCP< std::vector<Element_Linear2D> > cells = mb.myElements();
      for (std::vector<Element_Linear2D>::iterator cell = cells->begin();
	   cell != cells->end(); ++cell) {
	
	const shards::Array<double,shards::NaturalOrder,Node,Dim>& coords = 
	  cell->nodeCoordinates();

	for (int node=0; node < cell->numNodes(); ++node) {
	  coordinates[cell->globalNodeId(node)].resize(dim);
	  coordinates[cell->globalNodeId(node)][0] = coords(node,0);
	  coordinates[cell->globalNodeId(node)][1] = coords(node,1);
	}	
      }


      {
	std::vector< RCP<ofstream> > files; 
	for (std::size_t eq = 0; eq < num_eq; ++eq) {
	  std::stringstream ost;
	  ost << "upper_DOF" << eq << "_PID" << comm->MyPID() << ".dat";
	  files.push_back( rcp(new std::ofstream(ost.str().c_str()), 
			       ios::out | ios::trunc) );
	  files[eq]->precision(10);
	}
	
	const std::vector<int>& node_list = mb.topNodeSetGlobalIds();
	for (std::size_t node = 0; node < node_list.size(); ++node) {
	  int lid = owned_x->Map().LID(node_list[node] * num_eq);
	  for (std::size_t eq = 0; eq < num_eq; ++eq) {
	    int dof_index = lid + eq;
	    *(files[eq]) << coordinates[node_list[node]][0] << "   " 
			 << (*owned_x)[dof_index] << endl;
	  }
	}
      }

      {
	std::vector< RCP<ofstream> > files; 
	for (std::size_t eq = 0; eq < num_eq; ++eq) {
	  std::stringstream ost;
	  ost << "lower_DOF" << eq << "_PID" << comm->MyPID() << ".dat";
	  files.push_back( rcp(new std::ofstream(ost.str().c_str()), 
			       ios::out | ios::trunc) );
	  files[eq]->precision(10);
	}
	
	const std::vector<int>& node_list = mb.bottomNodeSetGlobalIds();
	for (std::size_t node = 0; node < node_list.size(); ++node) {
	  int lid = owned_x->Map().LID(node_list[node] * num_eq);
	  for (std::size_t eq = 0; eq < num_eq; ++eq) {
	    int dof_index = lid + eq;
	    *(files[eq]) << coordinates[node_list[node]][0] << "   " 
			 << (*owned_x)[dof_index] << endl;
	  }
	}
      }

    }

    TEUCHOS_TEST_FOR_EXCEPTION(!converged, std::runtime_error,
		       "Problem failed to converge!");

    TEUCHOS_TEST_FOR_EXCEPTION(num_newton_steps != 10, std::runtime_error,
		       "Incorrect number of Newton steps!");

    // Only check num gmres steps in serial
#ifndef HAVE_MPI
    TEUCHOS_TEST_FOR_EXCEPTION(num_gmres_iterations != 10, std::runtime_error,
		       "Incorrect number of GMRES iterations!");
#endif


    */

    // *********************************************************************
    // Finished all testing
    // *********************************************************************
    if (comm->MyPID() == 0)
      std::cout << "\nRun has completed successfully!\n" << std::endl; 
    // *********************************************************************
    // *********************************************************************

  }
  catch (const std::exception& e) {
    std::cout << "************************************************" << endl;
    std::cout << "************************************************" << endl;
    std::cout << "Exception Caught!" << endl;
    std::cout << "Error message is below\n " << e.what() << endl;
    std::cout << "************************************************" << endl;
  }
  catch (...) {
    std::cout << "************************************************" << endl;
    std::cout << "************************************************" << endl;
    std::cout << "Unknown Exception Caught!" << endl;
    std::cout << "************************************************" << endl;
  }

  TimeMonitor::summarize();
    
#ifdef HAVE_MPI
  MPI_Finalize();
#endif

  return 0;
}
int main_(Teuchos::CommandLineProcessor &clp, int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::TimeMonitor;

  // =========================================================================
  // MPI initialization using Teuchos
  // =========================================================================
  Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);

  RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
  int numProc = comm->getSize();
  int myRank  = comm->getRank();

  // =========================================================================
  // Parameters initialization
  // =========================================================================
  ::Xpetra::Parameters xpetraParameters(clp);

  bool runHeavyTests = false;
  clp.setOption("heavytests", "noheavytests",  &runHeavyTests, "whether to exercise tests that take a long time to run");

  clp.recogniseAllOptions(true);
  switch (clp.parse(argc,argv)) {
    case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED:        return EXIT_SUCCESS;
    case Teuchos::CommandLineProcessor::PARSE_ERROR:
    case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
    case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL:          break;
  }

  Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();

  // =========================================================================
  // Problem construction
  // =========================================================================
  ParameterList matrixParameters;
  matrixParameters.set("nx",         Teuchos::as<GO>(9999));
  matrixParameters.set("matrixType", "Laplace1D");
  RCP<Matrix>      A           = MueLuTests::TestHelpers::TestFactory<SC, LO, GO, NO>::Build1DPoisson(matrixParameters.get<GO>("nx"), lib);
  RCP<MultiVector> coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", A->getRowMap(), matrixParameters);

  std::string outDir = "Output/";

  std::vector<std::string> dirList;
  if (runHeavyTests) {
    dirList.push_back("EasyParameterListInterpreter-heavy/");
    dirList.push_back("FactoryParameterListInterpreter-heavy/");
  } else {
    dirList.push_back("EasyParameterListInterpreter/");
    dirList.push_back("FactoryParameterListInterpreter/");
  }
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ISORROPIA) && defined(HAVE_AMESOS2_KLU2)
  // The ML interpreter have internal ifdef, which means that the resulting
  // output would depend on configuration (reguarl interpreter does not have
  // that). Therefore, we need to stabilize the configuration here.
  // In addition, we run ML parameter list tests only if KLU is available
  dirList.push_back("MLParameterListInterpreter/");
  dirList.push_back("MLParameterListInterpreter2/");
#endif
  int numLists = dirList.size();

  bool failed = false;
  Teuchos::Time timer("Interpreter timer");
  //double lastTime = timer.wallTime();
  for (int k = 0; k < numLists; k++) {
    Teuchos::ArrayRCP<std::string> fileList = MueLuTests::TestHelpers::GetFileList(dirList[k],
      (numProc == 1 ? std::string(".xml") : std::string("_np" + Teuchos::toString(numProc) + ".xml")));

    for (int i = 0; i < fileList.size(); i++) {
      // Set seed
      std::srand(12345);

      // Reset (potentially) cached value of the estimate
      A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());

      std::string xmlFile  = dirList[k] + fileList[i];
      std::string outFile  = outDir     + fileList[i];
      std::string baseFile = outFile.substr(0, outFile.find_last_of('.'));
      std::size_t found = baseFile.find("_np");
      if (numProc == 1 && found != std::string::npos) {
#ifdef HAVE_MPI
        baseFile = baseFile.substr(0, found);
#else
        std::cout << "Skipping \"" << xmlFile << "\" as MPI is not enabled" << std::endl;
        continue;
#endif
      }
      baseFile = baseFile + (lib == Xpetra::UseEpetra ? "_epetra" : "_tpetra");
      std::string goldFile = baseFile + ".gold";
      std::ifstream f(goldFile.c_str());
      if (!f.good()) {
        if (myRank == 0)
          std::cout << "Warning: comparison file " << goldFile << " not found.  Skipping test" << std::endl;
        continue;
      }

      std::filebuf    buffer;
      std::streambuf* oldbuffer = NULL;
      if (myRank == 0) {
        // Redirect output
        buffer.open((baseFile + ".out").c_str(), std::ios::out);
        oldbuffer = std::cout.rdbuf(&buffer);
      }

      // NOTE: we cannot use ParameterListInterpreter(xmlFile, comm), because we want to update the ParameterList
      // first to include "test" verbosity
      Teuchos::ParameterList paramList;
      Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFile, Teuchos::Ptr<Teuchos::ParameterList>(&paramList), *comm);
      if      (dirList[k] == "EasyParameterListInterpreter/" || dirList[k] == "EasyParameterListInterpreter-heavy/")
        paramList.set("verbosity", "test");
      else if (dirList[k] == "FactoryParameterListInterpreter/" || dirList[k] == "FactoryParameterListInterpreter-heavy/")
        paramList.sublist("Hierarchy").set("verbosity", "Test");
      else if (dirList[k] == "MLParameterListInterpreter/")
        paramList.set("ML output",     42);
      else if (dirList[k] == "MLParameterListInterpreter2/")
        paramList.set("ML output",     10);

      try {
        timer.start();
        Teuchos::RCP<HierarchyManager> mueluFactory;

        // create parameter list interpreter
        // here we have to distinguish between the general MueLu parameter list interpreter
        // and the ML parameter list interpreter. Note that the ML paramter interpreter also
        // works with Tpetra matrices.
        if (dirList[k] == "EasyParameterListInterpreter/"         ||
            dirList[k] == "EasyParameterListInterpreter-heavy/"   ||
            dirList[k] == "FactoryParameterListInterpreter/"      ||
            dirList[k] == "FactoryParameterListInterpreter-heavy/") {
          mueluFactory = Teuchos::rcp(new ParameterListInterpreter(paramList));

        } else if (dirList[k] == "MLParameterListInterpreter/") {
          mueluFactory = Teuchos::rcp(new MLParameterListInterpreter(paramList));

        } else if (dirList[k] == "MLParameterListInterpreter2/") {
          //std::cout << "ML ParameterList: " << std::endl;
          //std::cout << paramList << std::endl;
          RCP<ParameterList> mueluParamList = Teuchos::getParametersFromXmlString(MueLu::ML2MueLuParameterTranslator::translate(paramList,"SA"));
          //std::cout << "MueLu ParameterList: " << std::endl;
          //std::cout << *mueluParamList << std::endl;
          mueluFactory = Teuchos::rcp(new ParameterListInterpreter(*mueluParamList));
        }

        RCP<Hierarchy> H = mueluFactory->CreateHierarchy();

        H->GetLevel(0)->template Set<RCP<Matrix> >("A", A);

        if (dirList[k] == "MLParameterListInterpreter/") {
          // MLParameterInterpreter needs the nullspace information if rebalancing is active!
          // add default constant null space vector
          RCP<MultiVector> nullspace = MultiVectorFactory::Build(A->getRowMap(), 1);
          nullspace->putScalar(1.0);
          H->GetLevel(0)->Set("Nullspace", nullspace);
        }

        H->GetLevel(0)->Set("Coordinates", coordinates);

        mueluFactory->SetupHierarchy(*H);

        if (strncmp(fileList[i].c_str(), "reuse", 5) == 0) {
          // Build the Hierarchy the second time
          // Should be faster if we actually do the reuse
          A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());
          mueluFactory->SetupHierarchy(*H);
        }

        timer.stop();
      } catch (Teuchos::ExceptionBase& e) {
        std::string msg = e.what();
        msg = msg.substr(msg.find_last_of('\n')+1);

        if (myRank == 0) {
          std::cout << "Caught exception: " << msg << std::endl;

          // Redirect output back
          std::cout.rdbuf(oldbuffer);
          buffer.close();
        }

        if (msg == "Zoltan interface is not available" ||
            msg == "Zoltan2 interface is not available" ||
            msg == "MueLu::FactoryFactory:BuildFactory(): Cannot create a Zoltan2Interface object: Zoltan2 is disabled: HAVE_MUELU_ZOLTAN2 && HAVE_MPI == false.") {

          if (myRank == 0)
            std::cout << xmlFile << ": skipped (missing library)" << std::endl;

          continue;
        }
      }

      std::string cmd;
      if (myRank == 0) {
        // Redirect output back
        std::cout.rdbuf(oldbuffer);
        buffer.close();

        // Create a copy of outputs
        cmd = "cp -f ";
        system((cmd + baseFile + ".gold " + baseFile + ".gold_filtered").c_str());
        system((cmd + baseFile + ".out " + baseFile + ".out_filtered").c_str());

        // Tpetra produces different eigenvalues in Chebyshev due to using
        // std::rand() for generating random vectors, which may be initialized
        // using different seed, and may have different algorithm from one
        // gcc version to another, or to anogther compiler (like clang)
        // This leads to us always failing this test.
        // NOTE1 : Epetra, on the other hand, rolls out its out random number
        // generator, which always produces same results

        // Ignore the value of "lambdaMax"
        run_sed("'s/lambdaMax: [0-9]*.[0-9]*/lambdaMax = <ignored>/'", baseFile);

        // Ignore the value of "lambdaMin"
        run_sed("'s/lambdaMin: [0-9]*.[0-9]*/lambdaMin = <ignored>/'", baseFile);

        // Ignore the value of "chebyshev: max eigenvalue"
        // NOTE: we skip lines with default value ([default])
        run_sed("'/[default]/! s/chebyshev: max eigenvalue = [0-9]*.[0-9]*/chebyshev: max eigenvalue = <ignored>/'", baseFile);

        // Ignore the exact type of direct solver (it is selected semi-automatically
        // depending on how Trilinos was configured
        run_sed("'s/Amesos\\([2]*\\)Smoother{type = .*}/Amesos\\1Smoother{type = <ignored>}/'", baseFile);
        run_sed("'s/SuperLU solver interface, direct solve/<Direct> solver interface/'", baseFile);
        run_sed("'s/KLU2 solver interface/<Direct> solver interface/'", baseFile);
        run_sed("'s/Basker solver interface/<Direct> solver interface/'", baseFile);

        // Strip template args for some classes
        std::vector<std::string> classes;
        classes.push_back("Xpetra::Matrix");
        classes.push_back("MueLu::Constraint");
        classes.push_back("MueLu::SmootherPrototype");
        for (size_t q = 0; q < classes.size(); q++)
          run_sed("'s/" + classes[q] + "<.*>/" + classes[q] + "<ignored> >/'", baseFile);

#ifdef __APPLE__
        // Some Macs print outs ptrs as 0x0 instead of 0, fix that
        run_sed("'/RCP/ s/=0x0/=0/g'", baseFile);
#endif

        // Run comparison (ignoring whitespaces)
        cmd = "diff -u -w -I\"^\\s*$\" " + baseFile + ".gold_filtered " + baseFile + ".out_filtered";
        int ret = system(cmd.c_str());
        if (ret)
          failed = true;

        //std::ios_base::fmtflags ff(std::cout.flags());
        //std::cout.precision(2);
        //std::cout << xmlFile << " (" << std::setiosflags(std::ios::fixed)
        //          << timer.wallTime() - lastTime << " sec.) : " << (ret ? "failed" : "passed") << std::endl;
        //lastTime = timer.wallTime();
        //std::cout.flags(ff); // reset flags to whatever they were prior to printing time
        std::cout << xmlFile << " : " << (ret ? "failed" : "passed") << std::endl;
      }
    }
  }

  if (myRank == 0)
    std::cout << std::endl << "End Result: TEST " << (failed ? "FAILED" : "PASSED") << std::endl;

  return (failed ? EXIT_FAILURE : EXIT_SUCCESS);
}
int main(int argc, char *argv[])
{
    using std::cout;
    using std::endl;
    int MyPID = 0;
    bool boolret;

#ifdef HAVE_MPI
    // Initialize MPI
    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &MyPID);
#endif


    bool testFailed;
    bool verbose = false;
    bool debug = false;
    std::string which("LM");

    CommandLineProcessor cmdp(false,true);
    cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
    cmdp.setOption("debug","nodebug",&debug,"Print debugging information.");
    cmdp.setOption("sort",&which,"Targetted eigenvalues (SM or LM).");
    if (cmdp.parse(argc,argv) != CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
        MPI_Finalize();
#endif
        return -1;
    }

#ifdef HAVE_COMPLEX
    typedef std::complex<double> ST;
#elif HAVE_COMPLEX_H
    typedef ::complex<double> ST;
#else
    typedef double ST;
    // no complex. quit with failure.
    if (verbose && MyPID == 0) {
        cout << "Not compiled with complex support." << endl;
        cout << "End Result: TEST FAILED" << endl;
#ifdef HAVE_MPI
        MPI_Finalize();
#endif
        return -1;
    }
#endif
    typedef ScalarTraits<ST>                   SCT;
    typedef SCT::magnitudeType                  MT;
    typedef Anasazi::MultiVec<ST>               MV;
    typedef Anasazi::Operator<ST>               OP;
    typedef Anasazi::MultiVecTraits<ST,MV>     MVT;
    typedef Anasazi::OperatorTraits<ST,MV,OP>  OPT;
    ST ONE  = SCT::one();

    if (verbose && MyPID == 0) {
        cout << Anasazi::Anasazi_Version() << endl << endl;
    }

    // -- Set finite difference grid
    int dim = 10;

    // Build the problem matrix
    RCP< const MyOperator<ST> > K
        = rcp( new MyOperator<ST>(dim) );

    // Create initial vectors
    int blockSize = 2;
    RCP<MyMultiVec<ST> > ivec = rcp( new MyMultiVec<ST>(dim,blockSize) );
    ivec->MvRandom();

    // Create eigenproblem
    const int nev = 1;
    RCP<Anasazi::BasicEigenproblem<ST,MV,OP> > problem =
        rcp( new Anasazi::BasicEigenproblem<ST,MV,OP>(K,ivec) );
    //
    // Inform the eigenproblem that the operator K is non-Hermitian (even when it truly is Hermitian)
    problem->setHermitian(false);
    //
    // Set the number of eigenvalues requested
    problem->setNEV( nev );
    //
    // Inform the eigenproblem that you are done passing it information
    boolret = problem->setProblem();
    if (boolret != true) {
        if (verbose && MyPID == 0) {
            cout << "Anasazi::BasicEigenproblem::SetProblem() returned with error." << endl
                 << "End Result: TEST FAILED" << endl;
        }
#ifdef HAVE_MPI
        MPI_Finalize() ;
#endif
        return -1;
    }

    // Set verbosity level
    int verbosity = Anasazi::Errors + Anasazi::Warnings;
    if (verbose) {
        verbosity += Anasazi::FinalSummary + Anasazi::TimingDetails;
    }
    if (debug) {
        verbosity += Anasazi::Debug;
    }

    // Eigensolver parameters
    int numBlocks = 3;
    int maxRestarts = 100;
    MT tol = 1.0e-6;
    //
    // Create parameter list to pass into the solver manager
    ParameterList MyPL;
    MyPL.set( "Verbosity", verbosity );
    MyPL.set( "Which", which );
    MyPL.set( "Block Size", blockSize );
    MyPL.set( "Num Blocks", numBlocks );
    MyPL.set( "Maximum Restarts", maxRestarts );
    MyPL.set( "Convergence Tolerance", tol );
    //
    // Create the solver manager
    Anasazi::BlockKrylovSchurSolMgr<ST,MV,OP> MySolverMgr(problem, MyPL);

    // Solve the problem to the specified tolerances or length
    Anasazi::ReturnType returnCode = MySolverMgr.solve();
    testFailed = false;
    if (returnCode != Anasazi::Converged) {
        testFailed = true;
    }

    // Get the eigenvalues and eigenvectors from the eigenproblem
    Anasazi::Eigensolution<ST,MV> sol = problem->getSolution();
    RCP<MV> evecs = sol.Evecs;
    int numev = sol.numVecs;

    if (numev > 0) {

        std::ostringstream os;
        os.setf(std::ios::scientific, std::ios::floatfield);
        os.precision(6);

        // Compute the direct residual
        std::vector<MT> normV( numev );
        SerialDenseMatrix<int,ST> T(numev,numev);
        for (int i=0; i<numev; i++) {
            T(i,i) = sol.Evals[i].realpart;
        }
        RCP<MV> Kvecs = MVT::Clone( *evecs, numev );

        OPT::Apply( *K, *evecs, *Kvecs );

        MVT::MvTimesMatAddMv( -ONE, *evecs, T, ONE, *Kvecs );
        MVT::MvNorm( *Kvecs, normV );

        os << "Direct residual norms computed in BlockKrylovSchurComplex_test.exe" << endl
           << std::setw(20) << "Eigenvalue" << std::setw(20) << "Residual  " << endl
           << "----------------------------------------" << endl;
        for (int i=0; i<numev; i++) {
            if ( SCT::magnitude(sol.Evals[i].realpart) != SCT::zero() ) {
                normV[i] = SCT::magnitude(normV[i]/sol.Evals[i].realpart);
            }
            os << std::setw(20) << sol.Evals[i].realpart << std::setw(20) << normV[i] << endl;
            if ( normV[i] > tol ) {
                testFailed = true;
            }
        }
        if (verbose && MyPID==0) {
            cout << endl << os.str() << endl;
        }

    }

#ifdef HAVE_MPI
    MPI_Finalize() ;
#endif

    if (testFailed) {
        if (verbose && MyPID==0) {
            cout << "End Result: TEST FAILED" << endl;
        }
        return -1;
    }
    //
    // Default return value
    //
    if (verbose && MyPID==0) {
        cout << "End Result: TEST PASSED" << endl;
    }
    return 0;

}