int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
typedef Teuchos::ScalarTraits<SC> ST;
bool success = false;
bool verbose = true;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false);
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748);
Xpetra::Parameters xpetraParameters(clp);
bool optRecycling = true; clp.setOption("recycling", "no-recycling", &optRecycling, "Enable recycling of the multigrid preconditioner");
/* DO NOT WORK YET
bool optRecyclingRAPpattern = true; clp.setOption("recycling-rap-pattern", "no-recycling-rap-pattern", &optRecyclingRAPpattern, "Enable recycling of Ac=RAP pattern");
bool optRecyclingAPpattern = false; clp.setOption("recycling-ap-pattern", "no-recycling-ap-pattern", &optRecyclingAPpattern, "Enable recycling of AP pattern");
*/
bool optRecyclingRAPpattern = false;
bool optRecyclingAPpattern = false;
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
// option dependencies
if (optRecycling == false) {
optRecyclingRAPpattern = false;
optRecyclingAPpattern = false;
}
//
// Construct the problems
//
RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A1 = Pr->BuildMatrix();
RCP<Matrix> A2 = Pr->BuildMatrix(); // TODO: generate another problem would be more meaningful (ex: scale A1)
//
// First solve
//
FactoryManager M;
Hierarchy H(A1);
if (optRecycling) {
// Configuring "Keep" options before running the first setup.
// Note: "Keep" flags should not be set on the default factories provided by the FactoryManager because in the current
// implementation of FactoryManager, those factories are freed and reallocated between Hierarchy::Setup() calls (see FactoryManager::Clean() and Hierarchy::Setup()).
// So we define our own factories here.
// AGGREGATES:
// Note: aggregates are only used to build Ptent, so it is not useful to keep them. Keeping Ptent is enough.
// RCP<Factory> AggFact = rcp(new CoupledAggregationFactory());
// M.SetFactory("Aggregates", AggFact);
// H.Keep("Aggregates", AggFact.get());
// PTENT:
RCP<Factory> PtentFact = rcp(new TentativePFactory());
M.SetFactory("Ptent", PtentFact);
H.Keep("P", PtentFact.get());
}
RCP<Factory> AcFact = rcp(new RAPFactory());
M.SetFactory("A", AcFact);
if (optRecyclingRAPpattern) {
H.Keep("RAP graph", AcFact.get());
}
if (optRecyclingAPpattern) {
H.Keep("AP graph", AcFact.get());
}
//
H.Setup(M);
{
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
X->putScalar((Scalar) 0.0);
B->setSeed(846930886); B->randomize();
int nIts = 9;
H.Iterate(*B, *X, nIts);
ST::magnitudeType residualNorms = Utilities::ResidualNorm(*A1, *X, *B)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
}
//
// Second solve
//
std::cout << "Status of the preconditioner between runs:" << std::endl;
H.print(*getFancyOStream(Teuchos::rcpFromRef(std::cout)), MueLu::High);
// Change the problem
RCP<Level> finestLevel = H.GetLevel(0);
finestLevel->Set("A", A2);
if (optRecycling) {
// Optional: this makes sure that the aggregates are never requested (and built) during the second run.
// If someone request the aggregates, an exception will be thrown.
M.SetFactory("Aggregates", MueLu::NoFactory::getRCP());
}
// Redo the setup
H.Setup(M);
{
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
X->putScalar((Scalar) 0.0);
B->setSeed(846930886); B->randomize();
int nIts = 9;
H.Iterate(*B, *X, nIts);
ST::magnitudeType residualNorms = Utilities::ResidualNorm(*A2, *X, *B)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
}
//
// Clean-up
//
// Remove kept data from the preconditioner. This will force recomputation on future runs. "Keep" flags are also removed.
if (optRecycling) {
//if aggregates explicitly kept: H.Delete("Aggregates", M.GetFactory("Aggregates").get());
H.Delete("P", M.GetFactory("Ptent").get());
}
if (optRecyclingRAPpattern) {
H.Delete("RAP graph", M.GetFactory("A").get());
}
if (optRecyclingAPpattern) {
H.Delete("AP graph", M.GetFactory("A").get());
}
std::cout << "Status of the preconditioner at the end:" << std::endl;
H.print(*getFancyOStream(Teuchos::rcpFromRef(std::cout)), MueLu::High);
success = true;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP; using Teuchos::rcp;
using Teuchos::TimeMonitor;
//using Galeri::Xpetra::CreateCartesianCoordinates;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// out->setOutputToRootOnly(-1);
// out->precision(12);
//FIXME we need a HAVE_MUELU_LONG_LONG_INT option
//#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
//#endif
//
// SET TEST PARAMETERS
//
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561 = 3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748 = 4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// Custom command line parameters
// - Debug
int optDebug = 0; clp.setOption("debug", &optDebug, "pause to attach debugger");
int optDump = 0; clp.setOption("dump", &optDump, "write matrix to file");
int optTimings = 0; clp.setOption("timings", &optTimings, "print timings to screen");
// - Levels
LO optMaxLevels = 10; clp.setOption("maxLevels", &optMaxLevels, "maximum number of levels allowed");
int optMaxCoarseSize = 50; clp.setOption("maxCoarseSize", &optMaxCoarseSize, "maximum #dofs in coarse operator"); //FIXME clp doesn't like long long int
// - Smoothed-Aggregation
Scalar optSaDamping = 4./3; clp.setOption("saDamping", &optSaDamping, "prolongator damping factor");
// - Aggregation
std::string optAggOrdering = "natural"; clp.setOption("aggOrdering", &optAggOrdering, "aggregation ordering strategy (natural, random, graph)");
int optMinPerAgg = 2; clp.setOption("minPerAgg", &optMinPerAgg, "minimum #DOFs per aggregate");
int optMaxNbrSel = 0; clp.setOption("maxNbrSel", &optMaxNbrSel, "maximum # of nbrs allowed to be in other aggregates");
// - R
int optExplicitR = 1; clp.setOption("explicitR", &optExplicitR, "restriction will be explicitly stored as transpose of prolongator");
// - Smoothers
std::string optSmooType = "sgs"; clp.setOption("smooType", &optSmooType, "smoother type ('l1-sgs', 'sgs 'or 'cheby')");
int optSweeps = 2; clp.setOption("sweeps", &optSweeps, "sweeps to be used in SGS (or Chebyshev degree)");
// - Repartitioning
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
int optRepartition = 1; clp.setOption("repartition", &optRepartition, "enable repartitioning (0=no repartitioning, 1=Zoltan RCB, 2=Isorropia+Zoltan PHG");
LO optMinRowsPerProc = 2000; clp.setOption("minRowsPerProc", &optMinRowsPerProc, "min #rows allowable per proc before repartitioning occurs");
double optNnzImbalance = 1.2; clp.setOption("nnzImbalance", &optNnzImbalance, "max allowable nonzero imbalance before repartitioning occurs");
#else
int optRepartition = 0;
#endif // HAVE_MPI && HAVE_MUELU_ZOLTAN
// - Solve
int optFixPoint = 1; clp.setOption("fixPoint", &optFixPoint, "apply multigrid as solver");
int optPrecond = 1; clp.setOption("precond", &optPrecond, "apply multigrid as preconditioner");
LO optIts = 10; clp.setOption("its", &optIts, "number of multigrid cycles");
double optTol = 1e-7; clp.setOption("tol", &optTol, "stopping tolerance for Krylov method");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
if (optDebug) {
Utils::PauseForDebugger();
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
std::transform(optSmooType.begin(), optSmooType.end(), optSmooType.begin(), ::tolower);
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
if (comm->getRank() == 0) {
std::cout << xpetraParameters << matrixParameters;
// TODO: print custom parameters // Or use paramList::print()!
}
//
// CREATE INITIAL MATRIX */
//
RCP<const Map> map;
RCP<Matrix> A;
RCP<MultiVector> coordinates;
{
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build"));
map = MapFactory::Build(lib, matrixParameters.GetNumGlobalElements(), 0, comm);
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
A = Pr->BuildMatrix();
if (matrixParameters.GetMatrixType() == "Laplace1D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
}
//
//
//
// dump matrix to file
if (optDump) {
std::string fileName = "Amat.mm";
Utils::Write(fileName, *A);
}
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullspace->norm1(norms);
if (comm->getRank() == 0)
std::cout << "||NS|| = " << norms[0] << std::endl;
RCP<MueLu::Hierarchy<SC, LO, GO, NO, LMO> > H;
//
//
// SETUP
//
//
{
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup"));
//
// Hierarchy
//
H = rcp(new Hierarchy());
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
H->SetMaxCoarseSize((GO) optMaxCoarseSize);
//
// Finest level
//
RCP<Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A", A);
Finest->Set("Nullspace", nullspace);
Finest->Set("Coordinates", coordinates); //FIXME: XCoordinates, YCoordinates, ..
//
// FactoryManager
//
FactoryManager M;
//
//
// Aggregation
//
{
RCP<UncoupledAggregationFactory> AggregationFact = rcp(new UncoupledAggregationFactory());
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min DOFs per aggregate : " << optMinPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << optMaxNbrSel << std::endl;
AggregationFact->SetMinNodesPerAggregate(optMinPerAgg); //TODO should increase if run anything othpermRFacter than 1D
AggregationFact->SetMaxNeighAlreadySelected(optMaxNbrSel);
std::transform(optAggOrdering.begin(), optAggOrdering.end(), optAggOrdering.begin(), ::tolower);
if (optAggOrdering == "natural") {
*out << "aggregate ordering : NATURAL" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::NATURAL);
} else if (optAggOrdering == "random") {
*out << "aggregate ordering : RANDOM" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::RANDOM);
} else if (optAggOrdering == "graph") {
*out << "aggregate ordering : GRAPH" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::GRAPH);
} else {
std::string msg = "main: bad aggregation option """ + optAggOrdering + """.";
throw(MueLu::Exceptions::RuntimeError(msg));
}
//AggregationFact->SetPhase3AggCreation(0.5);
M.SetFactory("Aggregates", AggregationFact);
*out << "=============================================================================" << std::endl;
}
//
// Transfer
//
{
//
// Non rebalanced factories
//
RCP<SaPFactory> PFact = rcp(new SaPFactory());
PFact->SetDampingFactor(optSaDamping);
RCP<Factory> RFact = rcp(new TransPFactory());
RCP<RAPFactory> AFact = rcp(new RAPFactory());
AFact->setVerbLevel(Teuchos::VERB_HIGH);
if (!optExplicitR) {
H->SetImplicitTranspose(true);
ParameterList Aclist = *(AFact->GetValidParameterList());
Aclist.set("implicit transpose", true);
AFact->SetParameterList(Aclist);
if (comm->getRank() == 0) std::cout << "\n\n* ***** USING IMPLICIT RESTRICTION OPERATOR ***** *\n" << std::endl;
}
//
// Repartitioning (if needed)
//
if (optRepartition == 0) {
// No repartitioning
// Configure FactoryManager
M.SetFactory("P", PFact);
M.SetFactory("R", RFact);
M.SetFactory("A", AFact);
} else {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
// Repartitioning
// The Factory Manager will be configured to return the rebalanced versions of P, R, A by default.
// Everytime we want to use the non-rebalanced versions, we need to explicitly define the generating factory.
RFact->SetFactory("P", PFact);
//
AFact->SetFactory("P", PFact);
AFact->SetFactory("R", RFact);
// Transfer coordinates
RCP<CoordinatesTransferFactory> TransferCoordinatesFact = rcp(new CoordinatesTransferFactory());
AFact->AddTransferFactory(TransferCoordinatesFact); // FIXME REMOVE
// Compute partition (creates "Partition" object)
if(optRepartition == 1) { // use plain Zoltan Interface
} else if (optRepartition == 2) { // use Isorropia + Zoltan interface
}
// Repartitioning (creates "Importer" from "Partition")
RCP<Factory> RepartitionFact = rcp(new RepartitionFactory());
{
Teuchos::ParameterList paramList;
paramList.set("minRowsPerProcessor", optMinRowsPerProc);
paramList.set("nonzeroImbalance", optNnzImbalance);
RepartitionFact->SetParameterList(paramList);
}
RepartitionFact->SetFactory("A", AFact);
if(optRepartition == 1) {
RCP<Factory> ZoltanFact = rcp(new ZoltanInterface());
ZoltanFact->SetFactory("A", AFact);
ZoltanFact->SetFactory("Coordinates", TransferCoordinatesFact);
RepartitionFact->SetFactory("Partition", ZoltanFact);
}
else if(optRepartition == 2) {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ISORROPIA)
RCP<MueLu::IsorropiaInterface<LO, GO, NO, LMO> > isoInterface = rcp(new MueLu::IsorropiaInterface<LO, GO, NO, LMO>());
isoInterface->SetFactory("A", AFact);
// we don't need Coordinates here!
RepartitionFact->SetFactory("Partition", isoInterface);
#else
if (comm->getRank() == 0)
std::cout << "Please recompile Trilinos with Isorropia support enabled." << std::endl;
return EXIT_FAILURE;
#endif
}
// Reordering of the transfer operators
RCP<Factory> RebalancedPFact = rcp(new RebalanceTransferFactory());
RebalancedPFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Interpolation")));
RebalancedPFact->SetFactory("P", PFact);
RebalancedPFact->SetFactory("Coordinates", TransferCoordinatesFact);
RebalancedPFact->SetFactory("Nullspace", M.GetFactory("Ptent")); // TODO
RCP<Factory> RebalancedRFact = rcp(new RebalanceTransferFactory());
RebalancedRFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Restriction")));
RebalancedRFact->SetFactory("R", RFact);
// Compute Ac from rebalanced P and R
RCP<Factory> RebalancedAFact = rcp(new RebalanceAcFactory());
RebalancedAFact->SetFactory("A", AFact);
// Configure FactoryManager
M.SetFactory("A", RebalancedAFact);
M.SetFactory("P", RebalancedPFact);
M.SetFactory("R", RebalancedRFact);
M.SetFactory("Nullspace", RebalancedPFact);
M.SetFactory("Coordinates", RebalancedPFact);
M.SetFactory("Importer", RepartitionFact);
#else
TEUCHOS_TEST_FOR_EXCEPT(true);
#endif
} // optRepartition
} // Transfer
//
// Smoothers
//
{
std::string ifpackType;
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) optSweeps);
ifpackList.set("relaxation: damping factor", (SC) 1.0);
if (optSmooType == "sgs") {
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
}
else if (optSmooType == "l1-sgs") {
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
ifpackList.set("relaxation: use l1", true);
} else if (optSmooType == "cheby") {
ifpackType = "CHEBYSHEV";
ifpackList.set("chebyshev: degree", (LO) optSweeps);
if (matrixParameters.GetMatrixType() == "Laplace1D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 3);
}
else if (matrixParameters.GetMatrixType() == "Laplace2D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 7);
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 20);
}
// ifpackList.set("chebyshev: max eigenvalue", (double) -1.0);
// ifpackList.set("chebyshev: min eigenvalue", (double) 1.0);
}
RCP<SmootherPrototype> smootherPrototype = rcp(new TrilinosSmoother(ifpackType, ifpackList));
M.SetFactory("Smoother", rcp(new SmootherFactory(smootherPrototype)));
}
//
// Setup preconditioner
//
int startLevel = 0;
// std::cout << startLevel << " " << optMaxLevels << std::endl;
H->Setup(M, startLevel, optMaxLevels);
} // end of Setup TimeMonitor
/*{ // some debug output
// print out content of levels
std::cout << "FINAL CONTENT of multigrid levels" << std::endl;
for(LO l = 0; l < H->GetNumLevels(); l++) {
RCP<Level> coarseLevel = H->GetLevel(l);
coarseLevel->print(*out);
}
std::cout << "END FINAL CONTENT of multigrid levels" << std::endl;
} // end debug output*/
//
//
// SOLVE
//
//
// Define X, B
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
X->setSeed(846930886);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
B->norm2(norms);
B->scale(1.0/norms[0]);
//
// Use AMG directly as an iterative method
//
if (optFixPoint) {
X->putScalar( (SC) 0.0);
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 3 - Fixed Point Solve"));
H->IsPreconditioner(false);
H->Iterate(*B, *X, optIts);
} // optFixedPt
//
// Use AMG as a preconditioner in Belos
//
#ifdef HAVE_MUELU_BELOS
if (optPrecond) {
RCP<TimeMonitor> tm;
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Operator object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 100;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", optTol); // Relative convergence tolerance requested
//belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
{
TimeMonitor tm2(*TimeMonitor::getNewTimer("ScalingTest: 5bis - Belos Internal Solve"));
ret = solver->solve();
} // end of TimeMonitor
// Get the number of iterations for this solve.
if (comm->getRank() == 0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
// Compute actual residuals.
int numrhs = 1;
std::vector<double> actual_resids( numrhs ); //TODO: double?
std::vector<double> rhs_norm( numrhs );
RCP<MultiVector> resid = MultiVectorFactory::Build(map, numrhs);
typedef Belos::OperatorTraits<SC, MV, OP> OPT;
typedef Belos::MultiVecTraits<SC, MV> MVT;
OPT::Apply( *belosOp, *X, *resid );
MVT::MvAddMv( -1.0, *resid, 1.0, *B, *resid );
MVT::MvNorm( *resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
*out<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for ( int i = 0; i<numrhs; i++) {
double actRes = actual_resids[i]/rhs_norm[i];
*out<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
//if (actRes > tol) { badRes = true; }
}
} //try
catch(...) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged) {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
} else {
if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
}
tm = Teuchos::null;
} //if (optPrecond)
#endif // HAVE_MUELU_BELOS
//
// Timer final summaries
//
globalTimeMonitor = Teuchos::null; // stop this timer before summary
if (optTimings)
TimeMonitor::summarize();
//
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
/**********************************************************************************/
/* SET TEST PARAMETERS */
/**********************************************************************************/
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561=3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748=4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// custom parameters
int nSmoothers=2;
LO maxLevels = 3;
LO its=10;
std::string coarseSolver="ifpack2";
// std::string coarseSolver="amesos2";
int pauseForDebugger=0;
clp.setOption("nSmoothers",&nSmoothers,"number of Gauss-Seidel smoothers in the MergedSmootehrs");
clp.setOption("maxLevels",&maxLevels,"maximum number of levels allowed. If 1, then a MergedSmoother is used on the coarse grid");
clp.setOption("its",&its,"number of multigrid cycles");
clp.setOption("coarseSolver",&coarseSolver,"amesos2 or ifpack2 (Tpetra specific. Ignored for Epetra)");
clp.setOption("debug",&pauseForDebugger,"pause to attach debugger");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
if (comm->getRank() == 0) {
// matrixParameters.print();
// xpetraParameters.print();
// TODO: print custom parameters
}
if (pauseForDebugger) {
Utils::PauseForDebugger();
}
/**********************************************************************************/
/* CREATE INITIAL MATRIX */
/**********************************************************************************/
const RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
RCP<Matrix> Op = Pr->BuildMatrix();
#ifdef NEUMANN
// Tranform matrix to Neumann b.c.
// Essentially, we need to update two diagonal elements
// TODO: calls to getLocalRowView not really needed
Op->resumeFill();
Teuchos::ArrayView<const LO> indices;
Teuchos::ArrayView<const SC> values;
Teuchos::Array<SC> newValues(2, 0.0);
size_t myRank = Op->getRowMap()->getComm()->getRank();
size_t nCpus = Op->getRowMap()->getComm()->getSize();
if (myRank == 0) { // JG TODO: can we use rowMap->isNodeLocalElement(0) instead for more genericity?
//LO firstRow = 0;
newValues[0] = 1.0; newValues[1] = -1.0;
Op->getLocalRowView(0, indices, values);
Op->replaceLocalValues(0, indices, newValues);
}
if (myRank == nCpus-1) { // JG TODO: can we use rowMap->isNodeLocalElement(lastRow) instead for more genericity?
LO lastRow = Op->getNodeNumRows()-1;
newValues[0] = -1.0; newValues[1] = 1.0;
Op->getLocalRowView(lastRow, indices, values);
Op->replaceLocalValues(lastRow, indices, newValues);
}
Op->fillComplete();
#endif // NEUMANN
/**********************************************************************************/
/* */
/**********************************************************************************/
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
std::cout << "||NS|| = " << norms[0] << std::endl;
RCP<MueLu::Hierarchy<SC,LO,GO,NO,LMO> > H = rcp( new Hierarchy() );
H->SetDefaultVerbLevel(MueLu::Extreme);
H->IsPreconditioner(false);
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A", Op);
Finest->Set("Nullspace", nullSpace);
FactoryManager M;
M.SetFactory("Aggregates", rcp(new CoupledAggregationFactory()));
M.SetFactory("Ptent", rcp(new TentativePFactory()));
M.SetFactory("P", rcp(new SaPFactory()));
#ifdef EMIN
// Energy-minimization
RCP<PatternFactory> PatternFact = rcp(new PatternFactory());
#if 0
PatternFact->SetFactory("P", M.GetFactory("Ptent"));
#else
PatternFact->SetFactory("P", M.GetFactory("P"));
#endif
M.SetFactory("Ppattern", PatternFact);
RCP<EminPFactory> EminPFact = rcp(new EminPFactory());
EminPFact->SetFactory("P", M.GetFactory("Ptent"));
M.SetFactory("P", EminPFact);
RCP<NullspacePresmoothFactory> NullPreFact = rcp(new NullspacePresmoothFactory());
NullPreFact->SetFactory("Nullspace", M.GetFactory("Nullspace"));
M.SetFactory("Nullspace", NullPreFact);
#endif
RCP<SmootherPrototype> smooProto = gimmeMergedSmoother(nSmoothers, xpetraParameters.GetLib(), coarseSolver, comm->getRank());
M.SetFactory("Smoother", rcp(new SmootherFactory(smooProto)));
Teuchos::ParameterList status;
RCP<SmootherPrototype> coarseProto;
if (maxLevels != 1)
coarseProto = gimmeCoarseProto(xpetraParameters.GetLib(), coarseSolver, comm->getRank());
else
coarseProto = gimmeMergedSmoother(nSmoothers, xpetraParameters.GetLib(), coarseSolver, comm->getRank());
if (coarseProto == Teuchos::null)
return EXIT_FAILURE;
#ifdef NEUMANN
// Use coarse level projection solver
RCP<SmootherPrototype> projectedSolver = rcp(new ProjectorSmoother(coarseProto));
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(projectedSolver));
#else
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(coarseProto));
#endif
M.SetFactory("CoarseSolver", coarseSolveFact);
H->EnableGraphDumping("graph.dot", 2);
H->Setup(M, 0, maxLevels);
//if (comm->getRank() == 0) {
// std::cout << "======================\n Multigrid statistics \n======================" << std::endl;
// status.print(std::cout,Teuchos::ParameterList::PrintOptions().indent(2));
//}
// Define RHS
RCP<MultiVector> X = MultiVectorFactory::Build(map,1);
RCP<MultiVector> RHS = MultiVectorFactory::Build(map,1);
X->setSeed(846930886);
X->randomize();
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_true|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
Op->apply(*X,*RHS,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
// Use AMG directly as an iterative method
{
X->putScalar( (SC) 0.0);
H->Iterate(*RHS,its,*X);
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_" << std::setprecision(2) << its << "|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
}
return EXIT_SUCCESS;
}