Ejemplo n.º 1
0
int
main (int argc, char *argv[])
{
  using std::endl;
  using Teuchos::CommandLineProcessor;

  Teuchos::GlobalMPISession mpiSession (&argc, &argv, NULL);
  const int myRank = mpiSession.getRank();
  Teuchos::oblackholestream blackHole;
  std::ostream& out = (myRank == 0) ? std::cout : blackHole;

  bool verbose = true;
  CommandLineProcessor cmdp(false,true);
  cmdp.setOption ("verbose", "quiet", &verbose, "Print messages and results.");

  // Parse the command-line arguments.
  {
    const CommandLineProcessor::EParseCommandLineReturn parseResult =
      cmdp.parse (argc,argv);
    // If the caller asks us to print the documentation, let the
    // "test" pass trivially.
    if (parseResult == CommandLineProcessor::PARSE_HELP_PRINTED) {
      out << "End Result: TEST PASSED" << endl;
      return EXIT_SUCCESS;
    }
    TEUCHOS_TEST_FOR_EXCEPTION(parseResult != CommandLineProcessor::PARSE_SUCCESSFUL,
		       std::invalid_argument,
		       "Failed to parse command-line arguments");
  }

  // Only let the tester print if in verbose mode.
  GivensTester<int, double> tester (verbose ? out : blackHole);
  const bool success = tester.test ();

  if (success) {
    out << "End Result: TEST PASSED" << endl;
    return EXIT_SUCCESS;
  } else {
    out << "End Result: TEST FAILED" << endl;
    return EXIT_FAILURE;
  }
}
Ejemplo n.º 2
0
// This program reads two Maps from a file.  The Maps must have the
// same process counts as the input communicator.
int
main (int argc, char* argv[])
{
  using Teuchos::broadcast;
  using Teuchos::outArg;
  using Teuchos::RCP;
  using Teuchos::REDUCE_MIN;
  using Teuchos::reduceAll;
  using std::cerr;
  using std::cout;
  using std::endl;
  typedef Tpetra::Map<> map_type;
  // Reader must be templated on a CrsMatrix specialization
  typedef Tpetra::MatrixMarket::Reader<Tpetra::CrsMatrix<> > reader_type;

  Teuchos::GlobalMPISession mpiSession (&argc, &argv);
  const int myRank = mpiSession.getRank ();
  const int rootRank = 0;
  auto comm = Tpetra::DefaultPlatform::getDefaultPlatform ().getComm ();

  std::string mapFile1, mapFile2;
  bool tolerant = false;
  bool debug = false;
  {
    const bool throwExceptions = false;
    const bool recognizeAllOptions = true; // let Kokkos options through
    Teuchos::CommandLineProcessor cmdLineProc (throwExceptions, recognizeAllOptions);
    cmdLineProc.setOption ("mapFile1", &mapFile1, "Filename of first Map file, to read on Process 0");
    cmdLineProc.setOption ("mapFile2", &mapFile2, "Filename of second Map file, to read on Process 0");
    cmdLineProc.setOption ("tolerant", "strict", &tolerant, "Read Maps in tolerant mode");
    cmdLineProc.setOption ("debug", "release", &debug, "Enable debug output");

    auto result = cmdLineProc.parse (argc, argv);
    if (result != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
      if (result == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED) {
        return EXIT_SUCCESS; // printed help; now we're done
      }
      else if (result != Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION) {
        return EXIT_FAILURE; // some error that's not just an unrecognized option
      }
    }
  }

  // On Process 0, test whether the files exist.
  int lclFileExists = 0;
  int gblFileExists = 0;
  if (myRank == rootRank) {
    std::ifstream f1 (mapFile1);
    lclFileExists = f1.good () ? 1 : 0;
    gblFileExists = lclFileExists;
    f1.close ();
  }
  broadcast<int, int> (*comm, rootRank, outArg (gblFileExists));
  if (gblFileExists != 1) {
    if (myRank == rootRank) {
      cerr << "Cannot read the first Map file \"" << mapFile1 << "\"!" << endl;
    }
    return EXIT_FAILURE;
  }

  if (myRank == rootRank) {
    std::ifstream f2 (mapFile2);
    lclFileExists = f2.good ();
    gblFileExists = lclFileExists;
    f2.close ();
  }
  broadcast<int, int> (*comm, rootRank, outArg (gblFileExists));
  if (gblFileExists != 1) {
    if (myRank == rootRank) {
      cerr << "Cannot read the second Map file \"" << mapFile2 << "\"!" << endl;
    }
    return EXIT_FAILURE;
  }

  if (debug && myRank == rootRank) {
    cerr << "Both Map files are readable" << endl;
  }

  RCP<const map_type> map1;
  RCP<const map_type> map2;

  int lclSuccess = 0;
  int gblSuccess = 0;
  std::ostringstream errStrm;
  try {
    map1 = reader_type::readMapFile (mapFile1, comm, tolerant, debug);
    lclSuccess = 1;
  }
  catch (std::exception& e) {
    errStrm << "Process " << myRank << ": Caught exception while reading "
      "first Map file: " << e.what () << endl;
  }
  catch (...) {
    errStrm << "Process " << myRank << ": Caught exception not a subclass of "
      "std::exception, while reading first Map file" << endl;
  }
  if (map1.is_null ()) {
    errStrm << "Process " << myRank << ": Attempt to read first Map file "
      "returned null" << endl;
    lclSuccess = 0;
  }

  reduceAll<int, int> (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
  if (gblSuccess != 1) {
    Tpetra::Details::gathervPrint (std::cerr, errStrm.str (), *comm);
    return EXIT_FAILURE;
  }

  if (debug && myRank == rootRank) {
    cerr << "Read first Map file" << endl;
  }

  try {
    map2 = reader_type::readMapFile (mapFile2, comm, tolerant, debug);
    lclSuccess = 1;
  }
  catch (std::exception& e) {
    errStrm << "Process " << myRank << ": Caught exception while reading "
      "second Map file: " << e.what () << endl;
  }
  catch (...) {
    errStrm << "Process " << myRank << ": Caught exception not a subclass of "
      "std::exception, while reading second Map file" << endl;
  }
  if (map2.is_null ()) {
    errStrm << "Process " << myRank << ": Attempt to read second Map file "
      "returned null" << endl;
    lclSuccess = 0;
  }

  reduceAll<int, int> (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
  if (gblSuccess != 1) {
    Tpetra::Details::gathervPrint (std::cerr, errStrm.str (), *comm);
    return EXIT_FAILURE;
  }

  if (debug && myRank == rootRank) {
    cerr << "Read second Map file" << endl;
  }

  // At this point, we have two valid Maps.  Let's print stuff about them.
  const bool compat = map1->isCompatible (*map2);
  const bool same = map1->isSameAs (*map2);
  // This is a local (per MPI process) property.  Thus, we have to
  // all-reduce to find out whether it's true on all processes (which
  // is actually what we want to know).
  const bool locallyFitted = Tpetra::Details::isLocallyFitted (*map1, *map2);
  const int locallyFittedInt = locallyFitted ? 1 : 0;
  int globallyFittedInt = 0; // output argument
  reduceAll<int, int> (*comm, REDUCE_MIN, locallyFittedInt,
                       outArg (globallyFittedInt));
  const bool globallyFitted = (globallyFittedInt == 1);

  if (myRank == rootRank) {
    cout << "Maps compatible? " << (compat ? "YES" : "NO") << endl
         << "Maps same?       " << (same ? "YES" : "NO") << endl
         << "Maps fitted?     " << (globallyFitted ? "YES" : "NO") << endl;
  }
  return EXIT_SUCCESS;
}
int
main (int argc, char *argv[])
{
  using namespace TrilinosCouplings; // Yes, this means I'm lazy.

  using TpetraIntrepidPoissonExample::exactResidualNorm;
  using TpetraIntrepidPoissonExample::makeMatrixAndRightHandSide;
  using TpetraIntrepidPoissonExample::solveWithBelos;
  using TpetraIntrepidPoissonExample::solveWithBelosGPU;
  using IntrepidPoissonExample::makeMeshInput;
  using IntrepidPoissonExample::parseCommandLineArguments;
  using IntrepidPoissonExample::setCommandLineArgumentDefaults;
  using IntrepidPoissonExample::setMaterialTensorOffDiagonalValue;
  using IntrepidPoissonExample::setUpCommandLineArguments;
  using Tpetra::DefaultPlatform;
  using Teuchos::Comm;
  using Teuchos::outArg;
  using Teuchos::ParameterList;
  using Teuchos::parameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::rcpFromRef;
  using Teuchos::getFancyOStream;
  using Teuchos::FancyOStream;
  using std::endl;
  // Pull in typedefs from the example's namespace.
  typedef TpetraIntrepidPoissonExample::ST ST;
#ifdef HAVE_TRILINOSCOUPLINGS_MUELU
  typedef TpetraIntrepidPoissonExample::LO LO;
  typedef TpetraIntrepidPoissonExample::GO GO;
#endif // HAVE_TRILINOSCOUPLINGS_MUELU
  typedef TpetraIntrepidPoissonExample::Node Node;
  typedef Teuchos::ScalarTraits<ST> STS;
  typedef STS::magnitudeType MT;
  typedef Teuchos::ScalarTraits<MT> STM;
  typedef TpetraIntrepidPoissonExample::sparse_matrix_type sparse_matrix_type;
  typedef TpetraIntrepidPoissonExample::vector_type vector_type;
  typedef TpetraIntrepidPoissonExample::operator_type operator_type;

  bool success = true;
  try {
    Teuchos::oblackholestream blackHole;
    Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
    const int myRank = mpiSession.getRank ();
    //const int numProcs = mpiSession.getNProc ();

    // Get the default communicator and Kokkos Node instance
    RCP<const Comm<int> > comm =
      DefaultPlatform::getDefaultPlatform ().getComm ();
    RCP<Node> node = DefaultPlatform::getDefaultPlatform ().getNode ();

    // Did the user specify --help at the command line to print help
    // with command-line arguments?
    bool printedHelp = false;
    // Values of command-line arguments.
    int nx, ny, nz;
    std::string xmlInputParamsFile;
    bool verbose, debug;
    int maxNumItersFromCmdLine = -1; // -1 means "read from XML file"
    double tolFromCmdLine = -1.0; // -1 means "read from XML file"
    std::string solverName = "GMRES";
    ST materialTensorOffDiagonalValue = 0.0;

    // Set default values of command-line arguments.
    setCommandLineArgumentDefaults (nx, ny, nz, xmlInputParamsFile,
                                    solverName, verbose, debug);
    // Parse and validate command-line arguments.
    Teuchos::CommandLineProcessor cmdp (false, true);
    setUpCommandLineArguments (cmdp, nx, ny, nz, xmlInputParamsFile,
                               solverName, tolFromCmdLine,
                               maxNumItersFromCmdLine,
                               verbose, debug);
    cmdp.setOption ("materialTensorOffDiagonalValue",
                    &materialTensorOffDiagonalValue, "Off-diagonal value in "
                    "the material tensor.  This controls the iteration count.  "
                    "Be careful with this if you use CG, since you can easily "
                    "make the matrix indefinite.");

    // Additional command-line arguments for GPU experimentation.
    bool gpu = false;
    cmdp.setOption ("gpu", "no-gpu", &gpu,
                    "Run example using GPU node (if supported)");
    int ranks_per_node = 1;
    cmdp.setOption ("ranks_per_node", &ranks_per_node,
                    "Number of MPI ranks per node");
    int gpu_ranks_per_node = 1;
    cmdp.setOption ("gpu_ranks_per_node", &gpu_ranks_per_node,
                    "Number of MPI ranks per node for GPUs");
    int device_offset = 0;
    cmdp.setOption ("device_offset", &device_offset,
                    "Offset for attaching MPI ranks to CUDA devices");

    // Additional command-line arguments for dumping the generated
    // matrix or its row Map to output files.
    //
    // FIXME (mfh 09 Apr 2014) Need to port these command-line
    // arguments to the Epetra version.

    // If matrixFilename is nonempty, dump the matrix to that file
    // in MatrixMarket format.
    std::string matrixFilename;
    cmdp.setOption ("matrixFilename", &matrixFilename, "If nonempty, dump the "
                    "generated matrix to that file in MatrixMarket format.");

    // If rowMapFilename is nonempty, dump the matrix's row Map to
    // that file in MatrixMarket format.
    std::string rowMapFilename;
    cmdp.setOption ("rowMapFilename", &rowMapFilename, "If nonempty, dump the "
                    "generated matrix's row Map to that file in a format that "
                    "Tpetra::MatrixMarket::Reader can read.");
    // Option to exit after building A and b (and dumping stuff to
    // files, if requested).
    bool exitAfterAssembly = false;
    cmdp.setOption ("exitAfterAssembly", "dontExitAfterAssembly",
                    &exitAfterAssembly, "If true, exit after building the "
                    "sparse matrix and dense right-hand side vector.  If either"
                    " --matrixFilename or --rowMapFilename are nonempty strings"
                    ", dump the matrix resp. row Map to their respective files "
                    "before exiting.");

    parseCommandLineArguments (cmdp, printedHelp, argc, argv, nx, ny, nz,
                               xmlInputParamsFile, solverName, verbose, debug);
    if (printedHelp) {
      // The user specified --help at the command line to print help
      // with command-line arguments.  We printed help already, so quit
      // with a happy return code.
      return EXIT_SUCCESS;
    }

    setMaterialTensorOffDiagonalValue (materialTensorOffDiagonalValue);

    // Both streams only print on MPI Rank 0.  "out" only prints if the
    // user specified --verbose.
    RCP<FancyOStream> out =
      getFancyOStream (rcpFromRef ((myRank == 0 && verbose) ? std::cout : blackHole));
    RCP<FancyOStream> err =
      getFancyOStream (rcpFromRef ((myRank == 0 && debug) ? std::cerr : blackHole));

#ifdef HAVE_MPI
    *out << "PARALLEL executable" << endl;
#else
    *out << "SERIAL executable" << endl;
#endif

    /**********************************************************************************/
    /********************************** GET XML INPUTS ********************************/
    /**********************************************************************************/
    ParameterList inputList;
    if (xmlInputParamsFile != "") {
      *out << "Reading parameters from XML file \""
           << xmlInputParamsFile << "\"..." << endl;
      Teuchos::updateParametersFromXmlFile (xmlInputParamsFile,
                                            outArg (inputList));
      if (myRank == 0) {
        inputList.print (*out, 2, true, true);
        *out << endl;
      }
    }

    // Get Pamgen mesh definition string, either from the input
    // ParameterList or from our function that makes a cube and fills in
    // the number of cells along each dimension.
    std::string meshInput = inputList.get("meshInput", "");
    if (meshInput == "") {
      *out << "Generating mesh input string: nx = " << nx
           << ", ny = " << ny
           << ", nz = " << nz << endl;
      meshInput = makeMeshInput (nx, ny, nz);
    }

    // Total application run time
    {
      TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Time", total_time);

      RCP<sparse_matrix_type> A;
      RCP<vector_type> B, X_exact, X;
      {
        TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Assembly", total_assembly);
        makeMatrixAndRightHandSide (A, B, X_exact, X, comm, node, meshInput,
                                    out, err, verbose, debug);
      }

      // Optionally dump the matrix and/or its row Map to files.
      {
        typedef Tpetra::MatrixMarket::Writer<sparse_matrix_type> writer_type;
        if (matrixFilename != "") {
          writer_type::writeSparseFile (matrixFilename, A);
        }
        if (rowMapFilename != "") {
          writer_type::writeMapFile (rowMapFilename, * (A->getRowMap ()));
        }
      }

      if (exitAfterAssembly) {
        // Users might still be interested in assembly time.
        Teuchos::TimeMonitor::report (comm.ptr (), std::cout);
        return EXIT_SUCCESS;
      }

      const std::vector<MT> norms = exactResidualNorm (A, B, X_exact);
      // X_exact is the exact solution of the PDE, projected onto the
      // discrete mesh.  It may not necessarily equal the exact solution
      // of the linear system.
      *out << "||B - A*X_exact||_2 = " << norms[0] << endl
           << "||B||_2 = " << norms[1] << endl
           << "||A||_F = " << norms[2] << endl;

      // Setup preconditioner
      std::string prec_type = inputList.get ("Preconditioner", "None");
      RCP<operator_type> M;
      {
        TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Preconditioner Setup", total_prec);

        if (prec_type == "MueLu") {
#ifdef HAVE_TRILINOSCOUPLINGS_MUELU
          if (inputList.isSublist("MueLu")) {
            ParameterList mueluParams = inputList.sublist("MueLu");
            M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A,mueluParams);
          } else {
            M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A);
          }
#else // NOT HAVE_TRILINOSCOUPLINGS_MUELU
          TEUCHOS_TEST_FOR_EXCEPTION(
            prec_type == "MueLu", std::runtime_error, "Tpetra scaling example: "
            "In order to precondition with MueLu, you must have built Trilinos "
            "with the MueLu package enabled.");
#endif // HAVE_TRILINOSCOUPLINGS_MUELU
        }
      } // setup preconditioner

      // Get the convergence tolerance for each linear solve.
      // If the user provided a nonnegative value at the command
      // line, it overrides any value in the input ParameterList.
      MT tol = STM::squareroot (STM::eps ()); // default value
      if (tolFromCmdLine < STM::zero ()) {
        tol = inputList.get ("Convergence Tolerance", tol);
      } else {
        tol = tolFromCmdLine;
      }

      // Get the maximum number of iterations for each linear solve.
      // If the user provided a value other than -1 at the command
      // line, it overrides any value in the input ParameterList.
      int maxNumIters = 200; // default value
      if (maxNumItersFromCmdLine == -1) {
        maxNumIters = inputList.get ("Maximum Iterations", maxNumIters);
      } else {
        maxNumIters = maxNumItersFromCmdLine;
      }

      // Get the number of "time steps."  We imitate a time-dependent
      // PDE by doing this many linear solves.
      const int num_steps = inputList.get ("Number of Time Steps", 1);

      // Do the linear solve(s).
      bool converged = false;
      int numItersPerformed = 0;
      if (gpu) {
        TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total GPU Solve", total_solve);
        solveWithBelosGPU (converged, numItersPerformed, tol, maxNumIters,
                           num_steps, ranks_per_node, gpu_ranks_per_node,
                           device_offset, prec_type, X, A, B, Teuchos::null, M);
      }
      else {
        TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Solve", total_solve);
        solveWithBelos (converged, numItersPerformed, solverName, tol,
                        maxNumIters, num_steps, X, A, B, Teuchos::null, M);
      }

      // Compute ||X-X_exact||_2
      const MT norm_x = X_exact->norm2 ();
      X_exact->update (-1.0, *X, 1.0);
      const MT norm_error = X_exact->norm2 ();
      *out << endl
           << "||X - X_exact||_2 / ||X_exact||_2 = " << norm_error / norm_x
           << endl;
    } // total time block

    // Summarize timings
    Teuchos::TimeMonitor::report (comm.ptr (), std::cout);
  } // try
  TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);

  if (success) {
    return EXIT_SUCCESS;
  } else {
    return EXIT_FAILURE;
  }
}
int
main (int argc, char *argv[])
{
  using namespace TrilinosCouplings; // Yes, this means I'm lazy.

  using TpetraIntrepidPoissonExample::exactResidualNorm;
  using TpetraIntrepidPoissonExample::makeMatrixAndRightHandSide;
  using TpetraIntrepidPoissonExample::solveWithBelos;
  using TpetraIntrepidPoissonExample::solveWithBelosGPU;
  using IntrepidPoissonExample::makeMeshInput;
  using IntrepidPoissonExample::setCommandLineArgumentDefaults;
  using IntrepidPoissonExample::setUpCommandLineArguments;
  using IntrepidPoissonExample::parseCommandLineArguments;
  using Tpetra::DefaultPlatform;
  using Teuchos::Comm;
  using Teuchos::outArg;
  using Teuchos::ParameterList;
  using Teuchos::parameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::rcpFromRef;
  using Teuchos::getFancyOStream;
  using Teuchos::FancyOStream;
  using std::endl;
  // Pull in typedefs from the example's namespace.
  typedef TpetraIntrepidPoissonExample::ST ST;
  typedef TpetraIntrepidPoissonExample::LO LO;
  typedef TpetraIntrepidPoissonExample::GO GO;
  typedef TpetraIntrepidPoissonExample::Node Node;
  typedef Teuchos::ScalarTraits<ST> STS;
  typedef STS::magnitudeType MT;
  typedef Teuchos::ScalarTraits<MT> STM;
  typedef TpetraIntrepidPoissonExample::sparse_matrix_type sparse_matrix_type;
  typedef TpetraIntrepidPoissonExample::vector_type vector_type;
  typedef TpetraIntrepidPoissonExample::operator_type operator_type;

  bool success = true;
  try {

  Teuchos::oblackholestream blackHole;
  Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
  const int myRank = mpiSession.getRank ();
  //const int numProcs = mpiSession.getNProc ();

  // Get the default communicator and Kokkos Node instance
  RCP<const Comm<int> > comm =
    DefaultPlatform::getDefaultPlatform ().getComm ();
  RCP<Node> node = DefaultPlatform::getDefaultPlatform ().getNode ();

  // Did the user specify --help at the command line to print help
  // with command-line arguments?
  bool printedHelp = false;
  // Values of command-line arguments.
  int nx, ny, nz;
  std::string xmlInputParamsFile;
  bool verbose, debug;

  // Set default values of command-line arguments.
  setCommandLineArgumentDefaults (nx, ny, nz, xmlInputParamsFile,
                                  verbose, debug);
  // Parse and validate command-line arguments.
  Teuchos::CommandLineProcessor cmdp (false, true);
  setUpCommandLineArguments (cmdp, nx, ny, nz, xmlInputParamsFile,
                             verbose, debug);
  bool gpu = false;
  cmdp.setOption ("gpu", "no-gpu", &gpu,
                  "Run example using GPU node (if supported)");
  int ranks_per_node = 1;
  cmdp.setOption("ranks_per_node", &ranks_per_node,
                 "Number of MPI ranks per node");
  int gpu_ranks_per_node = 1;
  cmdp.setOption("gpu_ranks_per_node", &gpu_ranks_per_node,
                 "Number of MPI ranks per node for GPUs");
  int device_offset = 0;
  cmdp.setOption("device_offset", &device_offset,
                 "Offset for attaching MPI ranks to CUDA devices");
  parseCommandLineArguments (cmdp, printedHelp, argc, argv, nx, ny, nz,
                             xmlInputParamsFile, verbose, debug);
  if (printedHelp) {
    // The user specified --help at the command line to print help
    // with command-line arguments.  We printed help already, so quit
    // with a happy return code.
    return EXIT_SUCCESS;
  }

  // Both streams only print on MPI Rank 0.  "out" only prints if the
  // user specified --verbose.
  RCP<FancyOStream> out =
    getFancyOStream (rcpFromRef ((myRank == 0 && verbose) ? std::cout : blackHole));
  RCP<FancyOStream> err =
    getFancyOStream (rcpFromRef ((myRank == 0 && debug) ? std::cerr : blackHole));

#ifdef HAVE_MPI
  *out << "PARALLEL executable" << endl;
#else
  *out << "SERIAL executable" << endl;
#endif

/**********************************************************************************/
/********************************** GET XML INPUTS ********************************/
/**********************************************************************************/
  ParameterList inputList;
  if (xmlInputParamsFile != "") {
    *out << "Reading parameters from XML file \""
         << xmlInputParamsFile << "\"..." << endl;
    Teuchos::updateParametersFromXmlFile (xmlInputParamsFile,
                                          outArg (inputList));
    if (myRank == 0) {
      inputList.print (*out, 2, true, true);
      *out << endl;
    }
  }

  // Get Pamgen mesh definition string, either from the input
  // ParameterList or from our function that makes a cube and fills in
  // the number of cells along each dimension.
  std::string meshInput = inputList.get("meshInput", "");
  if (meshInput == "") {
    *out << "Generating mesh input string: nx = " << nx
         << ", ny = " << ny
         << ", nz = " << nz << endl;
    meshInput = makeMeshInput (nx, ny, nz);
  }

  // Total application run time
  {
  TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Time", total_time);

  RCP<sparse_matrix_type> A;
  RCP<vector_type> B, X_exact, X;
  {
    TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Assembly", total_assembly);
    makeMatrixAndRightHandSide (A, B, X_exact, X, comm, node, meshInput,
                                out, err, verbose, debug);
  }

  const std::vector<MT> norms = exactResidualNorm (A, B, X_exact);
  // X_exact is the exact solution of the PDE, projected onto the
  // discrete mesh.  It may not necessarily equal the exact solution
  // of the linear system.
  *out << "||B - A*X_exact||_2 = " << norms[0] << endl
       << "||B||_2 = " << norms[1] << endl
       << "||A||_F = " << norms[2] << endl;

  // Setup preconditioner
  std::string prec_type = inputList.get("Preconditioner", "None");
  RCP<operator_type> M;
  {
    TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Preconditioner Setup", total_prec);

    if (prec_type == "MueLu") {
      if (inputList.isSublist("MueLu")) {
        ParameterList mueluParams = inputList.sublist("MueLu");
        M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A,mueluParams);
      } else {
        M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A);
      }
    }
  }

  bool converged = false;
  int numItersPerformed = 0;
  const MT tol = inputList.get("Convergence Tolerance",
                               STM::squareroot (STM::eps ()));
  const int maxNumIters = inputList.get("Maximum Iterations", 200);
  const int num_steps = inputList.get("Number of Time Steps", 1);
  if (gpu) {
    TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total GPU Solve", total_solve);
    solveWithBelosGPU(converged, numItersPerformed, tol, maxNumIters, num_steps,
                      ranks_per_node, gpu_ranks_per_node, device_offset,
                      prec_type,
                      X, A, B, Teuchos::null, M);
  }
  else {
    TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Solve", total_solve);
    solveWithBelos (converged, numItersPerformed, tol, maxNumIters, num_steps,
                    X, A, B, Teuchos::null, M);
  }

  // Compute ||X-X_exact||_2
  MT norm_x = X_exact->norm2();
  X_exact->update(-1.0, *X, 1.0);
  MT norm_error = X_exact->norm2();
  *out << endl
       << "||X-X_exact||_2 / ||X_exact||_2 = " << norm_error / norm_x
       << endl;

  } // total time block

  // Summarize timings
  // RCP<ParameterList> reportParams = parameterList ("TimeMonitor::report");
  // reportParams->set ("Report format", std::string ("YAML"));
  // reportParams->set ("writeGlobalStats", true);
  // Teuchos::TimeMonitor::report (*out, reportParams);
  Teuchos::TimeMonitor::summarize(std::cout);

  } //try
  TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);

  if (success)
    return EXIT_SUCCESS;
  return EXIT_FAILURE;
}