int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP; using Teuchos::rcp;
using Teuchos::TimeMonitor;
//using Galeri::Xpetra::CreateCartesianCoordinates;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// out->setOutputToRootOnly(-1);
// out->precision(12);
//FIXME we need a HAVE_MUELU_LONG_LONG_INT option
//#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
//#endif
//
// SET TEST PARAMETERS
//
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561 = 3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748 = 4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// Custom command line parameters
// - Debug
int optDebug = 0; clp.setOption("debug", &optDebug, "pause to attach debugger");
int optDump = 0; clp.setOption("dump", &optDump, "write matrix to file");
int optTimings = 0; clp.setOption("timings", &optTimings, "print timings to screen");
// - Levels
LO optMaxLevels = 10; clp.setOption("maxLevels", &optMaxLevels, "maximum number of levels allowed");
int optMaxCoarseSize = 50; clp.setOption("maxCoarseSize", &optMaxCoarseSize, "maximum #dofs in coarse operator"); //FIXME clp doesn't like long long int
// - Smoothed-Aggregation
Scalar optSaDamping = 4./3; clp.setOption("saDamping", &optSaDamping, "prolongator damping factor");
// - Aggregation
std::string optAggOrdering = "natural"; clp.setOption("aggOrdering", &optAggOrdering, "aggregation ordering strategy (natural, random, graph)");
int optMinPerAgg = 2; clp.setOption("minPerAgg", &optMinPerAgg, "minimum #DOFs per aggregate");
int optMaxNbrSel = 0; clp.setOption("maxNbrSel", &optMaxNbrSel, "maximum # of nbrs allowed to be in other aggregates");
// - R
int optExplicitR = 1; clp.setOption("explicitR", &optExplicitR, "restriction will be explicitly stored as transpose of prolongator");
// - Smoothers
std::string optSmooType = "sgs"; clp.setOption("smooType", &optSmooType, "smoother type ('l1-sgs', 'sgs 'or 'cheby')");
int optSweeps = 2; clp.setOption("sweeps", &optSweeps, "sweeps to be used in SGS (or Chebyshev degree)");
// - Repartitioning
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
int optRepartition = 1; clp.setOption("repartition", &optRepartition, "enable repartitioning (0=no repartitioning, 1=Zoltan RCB, 2=Isorropia+Zoltan PHG");
LO optMinRowsPerProc = 2000; clp.setOption("minRowsPerProc", &optMinRowsPerProc, "min #rows allowable per proc before repartitioning occurs");
double optNnzImbalance = 1.2; clp.setOption("nnzImbalance", &optNnzImbalance, "max allowable nonzero imbalance before repartitioning occurs");
#else
int optRepartition = 0;
#endif // HAVE_MPI && HAVE_MUELU_ZOLTAN
// - Solve
int optFixPoint = 1; clp.setOption("fixPoint", &optFixPoint, "apply multigrid as solver");
int optPrecond = 1; clp.setOption("precond", &optPrecond, "apply multigrid as preconditioner");
LO optIts = 10; clp.setOption("its", &optIts, "number of multigrid cycles");
double optTol = 1e-7; clp.setOption("tol", &optTol, "stopping tolerance for Krylov method");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
if (optDebug) {
Utils::PauseForDebugger();
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
std::transform(optSmooType.begin(), optSmooType.end(), optSmooType.begin(), ::tolower);
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
if (comm->getRank() == 0) {
std::cout << xpetraParameters << matrixParameters;
// TODO: print custom parameters // Or use paramList::print()!
}
//
// CREATE INITIAL MATRIX */
//
RCP<const Map> map;
RCP<Matrix> A;
RCP<MultiVector> coordinates;
{
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build"));
map = MapFactory::Build(lib, matrixParameters.GetNumGlobalElements(), 0, comm);
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
A = Pr->BuildMatrix();
if (matrixParameters.GetMatrixType() == "Laplace1D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
}
//
//
//
// dump matrix to file
if (optDump) {
std::string fileName = "Amat.mm";
Utils::Write(fileName, *A);
}
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullspace->norm1(norms);
if (comm->getRank() == 0)
std::cout << "||NS|| = " << norms[0] << std::endl;
RCP<MueLu::Hierarchy<SC, LO, GO, NO, LMO> > H;
//
//
// SETUP
//
//
{
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup"));
//
// Hierarchy
//
H = rcp(new Hierarchy());
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
H->SetMaxCoarseSize((GO) optMaxCoarseSize);
//
// Finest level
//
RCP<Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A", A);
Finest->Set("Nullspace", nullspace);
Finest->Set("Coordinates", coordinates); //FIXME: XCoordinates, YCoordinates, ..
//
// FactoryManager
//
FactoryManager M;
//
//
// Aggregation
//
{
RCP<UncoupledAggregationFactory> AggregationFact = rcp(new UncoupledAggregationFactory());
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min DOFs per aggregate : " << optMinPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << optMaxNbrSel << std::endl;
AggregationFact->SetMinNodesPerAggregate(optMinPerAgg); //TODO should increase if run anything othpermRFacter than 1D
AggregationFact->SetMaxNeighAlreadySelected(optMaxNbrSel);
std::transform(optAggOrdering.begin(), optAggOrdering.end(), optAggOrdering.begin(), ::tolower);
if (optAggOrdering == "natural") {
*out << "aggregate ordering : NATURAL" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::NATURAL);
} else if (optAggOrdering == "random") {
*out << "aggregate ordering : RANDOM" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::RANDOM);
} else if (optAggOrdering == "graph") {
*out << "aggregate ordering : GRAPH" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::GRAPH);
} else {
std::string msg = "main: bad aggregation option """ + optAggOrdering + """.";
throw(MueLu::Exceptions::RuntimeError(msg));
}
//AggregationFact->SetPhase3AggCreation(0.5);
M.SetFactory("Aggregates", AggregationFact);
*out << "=============================================================================" << std::endl;
}
//
// Transfer
//
{
//
// Non rebalanced factories
//
RCP<SaPFactory> PFact = rcp(new SaPFactory());
PFact->SetDampingFactor(optSaDamping);
RCP<Factory> RFact = rcp(new TransPFactory());
RCP<RAPFactory> AFact = rcp(new RAPFactory());
AFact->setVerbLevel(Teuchos::VERB_HIGH);
if (!optExplicitR) {
H->SetImplicitTranspose(true);
ParameterList Aclist = *(AFact->GetValidParameterList());
Aclist.set("implicit transpose", true);
AFact->SetParameterList(Aclist);
if (comm->getRank() == 0) std::cout << "\n\n* ***** USING IMPLICIT RESTRICTION OPERATOR ***** *\n" << std::endl;
}
//
// Repartitioning (if needed)
//
if (optRepartition == 0) {
// No repartitioning
// Configure FactoryManager
M.SetFactory("P", PFact);
M.SetFactory("R", RFact);
M.SetFactory("A", AFact);
} else {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
// Repartitioning
// The Factory Manager will be configured to return the rebalanced versions of P, R, A by default.
// Everytime we want to use the non-rebalanced versions, we need to explicitly define the generating factory.
RFact->SetFactory("P", PFact);
//
AFact->SetFactory("P", PFact);
AFact->SetFactory("R", RFact);
// Transfer coordinates
RCP<CoordinatesTransferFactory> TransferCoordinatesFact = rcp(new CoordinatesTransferFactory());
AFact->AddTransferFactory(TransferCoordinatesFact); // FIXME REMOVE
// Compute partition (creates "Partition" object)
if(optRepartition == 1) { // use plain Zoltan Interface
} else if (optRepartition == 2) { // use Isorropia + Zoltan interface
}
// Repartitioning (creates "Importer" from "Partition")
RCP<Factory> RepartitionFact = rcp(new RepartitionFactory());
{
Teuchos::ParameterList paramList;
paramList.set("minRowsPerProcessor", optMinRowsPerProc);
paramList.set("nonzeroImbalance", optNnzImbalance);
RepartitionFact->SetParameterList(paramList);
}
RepartitionFact->SetFactory("A", AFact);
if(optRepartition == 1) {
RCP<Factory> ZoltanFact = rcp(new ZoltanInterface());
ZoltanFact->SetFactory("A", AFact);
ZoltanFact->SetFactory("Coordinates", TransferCoordinatesFact);
RepartitionFact->SetFactory("Partition", ZoltanFact);
}
else if(optRepartition == 2) {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ISORROPIA)
RCP<MueLu::IsorropiaInterface<LO, GO, NO, LMO> > isoInterface = rcp(new MueLu::IsorropiaInterface<LO, GO, NO, LMO>());
isoInterface->SetFactory("A", AFact);
// we don't need Coordinates here!
RepartitionFact->SetFactory("Partition", isoInterface);
#else
if (comm->getRank() == 0)
std::cout << "Please recompile Trilinos with Isorropia support enabled." << std::endl;
return EXIT_FAILURE;
#endif
}
// Reordering of the transfer operators
RCP<Factory> RebalancedPFact = rcp(new RebalanceTransferFactory());
RebalancedPFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Interpolation")));
RebalancedPFact->SetFactory("P", PFact);
RebalancedPFact->SetFactory("Coordinates", TransferCoordinatesFact);
RebalancedPFact->SetFactory("Nullspace", M.GetFactory("Ptent")); // TODO
RCP<Factory> RebalancedRFact = rcp(new RebalanceTransferFactory());
RebalancedRFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Restriction")));
RebalancedRFact->SetFactory("R", RFact);
// Compute Ac from rebalanced P and R
RCP<Factory> RebalancedAFact = rcp(new RebalanceAcFactory());
RebalancedAFact->SetFactory("A", AFact);
// Configure FactoryManager
M.SetFactory("A", RebalancedAFact);
M.SetFactory("P", RebalancedPFact);
M.SetFactory("R", RebalancedRFact);
M.SetFactory("Nullspace", RebalancedPFact);
M.SetFactory("Coordinates", RebalancedPFact);
M.SetFactory("Importer", RepartitionFact);
#else
TEUCHOS_TEST_FOR_EXCEPT(true);
#endif
} // optRepartition
} // Transfer
//
// Smoothers
//
{
std::string ifpackType;
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) optSweeps);
ifpackList.set("relaxation: damping factor", (SC) 1.0);
if (optSmooType == "sgs") {
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
}
else if (optSmooType == "l1-sgs") {
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
ifpackList.set("relaxation: use l1", true);
} else if (optSmooType == "cheby") {
ifpackType = "CHEBYSHEV";
ifpackList.set("chebyshev: degree", (LO) optSweeps);
if (matrixParameters.GetMatrixType() == "Laplace1D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 3);
}
else if (matrixParameters.GetMatrixType() == "Laplace2D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 7);
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 20);
}
// ifpackList.set("chebyshev: max eigenvalue", (double) -1.0);
// ifpackList.set("chebyshev: min eigenvalue", (double) 1.0);
}
RCP<SmootherPrototype> smootherPrototype = rcp(new TrilinosSmoother(ifpackType, ifpackList));
M.SetFactory("Smoother", rcp(new SmootherFactory(smootherPrototype)));
}
//
// Setup preconditioner
//
int startLevel = 0;
// std::cout << startLevel << " " << optMaxLevels << std::endl;
H->Setup(M, startLevel, optMaxLevels);
} // end of Setup TimeMonitor
/*{ // some debug output
// print out content of levels
std::cout << "FINAL CONTENT of multigrid levels" << std::endl;
for(LO l = 0; l < H->GetNumLevels(); l++) {
RCP<Level> coarseLevel = H->GetLevel(l);
coarseLevel->print(*out);
}
std::cout << "END FINAL CONTENT of multigrid levels" << std::endl;
} // end debug output*/
//
//
// SOLVE
//
//
// Define X, B
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
X->setSeed(846930886);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
B->norm2(norms);
B->scale(1.0/norms[0]);
//
// Use AMG directly as an iterative method
//
if (optFixPoint) {
X->putScalar( (SC) 0.0);
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 3 - Fixed Point Solve"));
H->IsPreconditioner(false);
H->Iterate(*B, *X, optIts);
} // optFixedPt
//
// Use AMG as a preconditioner in Belos
//
#ifdef HAVE_MUELU_BELOS
if (optPrecond) {
RCP<TimeMonitor> tm;
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Operator object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 100;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", optTol); // Relative convergence tolerance requested
//belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
{
TimeMonitor tm2(*TimeMonitor::getNewTimer("ScalingTest: 5bis - Belos Internal Solve"));
ret = solver->solve();
} // end of TimeMonitor
// Get the number of iterations for this solve.
if (comm->getRank() == 0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
// Compute actual residuals.
int numrhs = 1;
std::vector<double> actual_resids( numrhs ); //TODO: double?
std::vector<double> rhs_norm( numrhs );
RCP<MultiVector> resid = MultiVectorFactory::Build(map, numrhs);
typedef Belos::OperatorTraits<SC, MV, OP> OPT;
typedef Belos::MultiVecTraits<SC, MV> MVT;
OPT::Apply( *belosOp, *X, *resid );
MVT::MvAddMv( -1.0, *resid, 1.0, *B, *resid );
MVT::MvNorm( *resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
*out<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for ( int i = 0; i<numrhs; i++) {
double actRes = actual_resids[i]/rhs_norm[i];
*out<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
//if (actRes > tol) { badRes = true; }
}
} //try
catch(...) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged) {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
} else {
if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
}
tm = Teuchos::null;
} //if (optPrecond)
#endif // HAVE_MUELU_BELOS
//
// Timer final summaries
//
globalTimeMonitor = Teuchos::null; // stop this timer before summary
if (optTimings)
TimeMonitor::summarize();
//
return EXIT_SUCCESS;
}
void MLParameterListInterpreter<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::SetParameterList(const Teuchos::ParameterList & paramList_in) {
Teuchos::ParameterList paramList = paramList_in;
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); // TODO: use internal out (GetOStream())
//
// Read top-level of the parameter list
//
// hard-coded default values == ML defaults according to the manual
MUELU_READ_PARAM(paramList, "ML output", int, 0, verbosityLevel);
MUELU_READ_PARAM(paramList, "max levels", int, 10, maxLevels);
MUELU_READ_PARAM(paramList, "PDE equations", int, 1, nDofsPerNode);
MUELU_READ_PARAM(paramList, "coarse: max size", int, 128, maxCoarseSize);
MUELU_READ_PARAM(paramList, "aggregation: type", std::string, "Uncoupled", agg_type);
//MUELU_READ_PARAM(paramList, "aggregation: threshold", double, 0.0, agg_threshold);
MUELU_READ_PARAM(paramList, "aggregation: damping factor", double, (double)4/(double)3, agg_damping);
//MUELU_READ_PARAM(paramList, "aggregation: smoothing sweeps", int, 1, agg_smoothingsweeps);
MUELU_READ_PARAM(paramList, "aggregation: nodes per aggregate", int, 1, minPerAgg);
MUELU_READ_PARAM(paramList, "aggregation: keep Dirichlet bcs", bool, false, bKeepDirichletBcs); // This is a MueLu specific extension that does not exist in ML
MUELU_READ_PARAM(paramList, "aggregation: max neighbours already aggregated", int, 0, maxNbrAlreadySelected); // This is a MueLu specific extension that does not exist in ML
MUELU_READ_PARAM(paramList, "null space: type", std::string, "default vectors", nullspaceType);
MUELU_READ_PARAM(paramList, "null space: dimension", int, -1, nullspaceDim); // TODO: ML default not in documentation
MUELU_READ_PARAM(paramList, "null space: vectors", double*, NULL, nullspaceVec); // TODO: ML default not in documentation
MUELU_READ_PARAM(paramList, "energy minimization: enable", bool, false, bEnergyMinimization);
MUELU_READ_PARAM(paramList, "RAP: fix diagonal", bool, false, bFixDiagonal); // This is a MueLu specific extension that does not exist in ML
//
// Move smoothers/aggregation/coarse parameters to sublists
//
// ML allows to have level-specific smoothers/aggregation/coarse parameters at the top level of the list or/and defined in sublists:
// See also: ML Guide section 6.4.1, MueLu::CreateSublists, ML_CreateSublists
ParameterList paramListWithSubList;
MueLu::CreateSublists(paramList, paramListWithSubList);
paramList = paramListWithSubList; // swap
// std::cout << std::endl << "Parameter list after CreateSublists" << std::endl;
// std::cout << paramListWithSubList << std::endl;
//
// Validate parameter list
//
{
bool validate = paramList.get("ML validate parameter list", true); /* true = default in ML */
if (validate) {
#if defined(HAVE_MUELU_ML) && defined(HAVE_MUELU_EPETRA)
// Validate parameter list using ML validator
int depth = paramList.get("ML validate depth", 5); /* 5 = default in ML */
TEUCHOS_TEST_FOR_EXCEPTION(! ML_Epetra::ValidateMLPParameters(paramList, depth), Exceptions::RuntimeError,
"ERROR: ML's Teuchos::ParameterList contains incorrect parameter!");
#else
// If no validator available: issue a warning and set parameter value to false in the output list
*out << "Warning: MueLu_ENABLE_ML=OFF. The parameter list cannot be validated." << std::endl;
paramList.set("ML validate parameter list", false);
#endif // HAVE_MUELU_ML
} // if(validate)
} // scope
//
//
//
// Matrix option
blksize_ = nDofsPerNode;
// Translate verbosity parameter
// Translate verbosity parameter
MsgType eVerbLevel = None;
if (verbosityLevel == 0) eVerbLevel = None;
if (verbosityLevel > 0) eVerbLevel = Low;
if (verbosityLevel > 4) eVerbLevel = Medium;
if (verbosityLevel > 7) eVerbLevel = High;
if (verbosityLevel > 9) eVerbLevel = Extreme;
if (verbosityLevel > 9) eVerbLevel = Test;
this->verbosity_ = eVerbLevel;
TEUCHOS_TEST_FOR_EXCEPTION(agg_type != "Uncoupled" && agg_type != "Coupled", Exceptions::RuntimeError, "MueLu::MLParameterListInterpreter::Setup(): parameter \"aggregation: type\": only 'Uncoupled' or 'Coupled' aggregation is supported.");
// Create MueLu factories
RCP<CoalesceDropFactory> dropFact = rcp(new CoalesceDropFactory());
RCP<FactoryBase> CoupledAggFact = Teuchos::null;
if(agg_type == "Uncoupled") {
// Uncoupled aggregation
RCP<UncoupledAggregationFactory> CoupledAggFact2 = rcp(new UncoupledAggregationFactory());
/*CoupledAggFact2->SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D
CoupledAggFact2->SetMaxNeighAlreadySelected(maxNbrAlreadySelected);
CoupledAggFact2->SetOrdering("natural");*/
CoupledAggFact2->SetFactory("Graph", dropFact);
CoupledAggFact2->SetFactory("DofsPerNode", dropFact);
CoupledAggFact2->SetParameter("UsePreserveDirichletAggregationAlgorithm", Teuchos::ParameterEntry(bKeepDirichletBcs));
CoupledAggFact2->SetParameter("aggregation: ordering", Teuchos::ParameterEntry(std::string("natural")));
CoupledAggFact2->SetParameter("aggregation: max selected neighbors", Teuchos::ParameterEntry(maxNbrAlreadySelected));
CoupledAggFact2->SetParameter("aggregation: min agg size", Teuchos::ParameterEntry(minPerAgg));
CoupledAggFact = CoupledAggFact2;
} else {
// Coupled Aggregation (default)
RCP<CoupledAggregationFactory> CoupledAggFact2 = rcp(new CoupledAggregationFactory());
CoupledAggFact2->SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D
CoupledAggFact2->SetMaxNeighAlreadySelected(maxNbrAlreadySelected);
CoupledAggFact2->SetOrdering("natural");
CoupledAggFact2->SetPhase3AggCreation(0.5);
CoupledAggFact2->SetFactory("Graph", dropFact);
CoupledAggFact2->SetFactory("DofsPerNode", dropFact);
CoupledAggFact = CoupledAggFact2;
}
if (verbosityLevel > 3) { // TODO fix me: Setup is a static function: we cannot use GetOStream without an object...
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min Nodes per aggregate : " << minPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << maxNbrAlreadySelected << std::endl;
*out << "aggregate ordering : natural" << std::endl;
*out << "=============================================================================" << std::endl;
}
RCP<Factory> PFact;
RCP<Factory> RFact;
RCP<Factory> PtentFact = rcp( new TentativePFactory() );
if (agg_damping == 0.0 && bEnergyMinimization == false) {
// tentative prolongation operator (PA-AMG)
PFact = PtentFact;
RFact = rcp( new TransPFactory() );
} else if (agg_damping != 0.0 && bEnergyMinimization == false) {
// smoothed aggregation (SA-AMG)
RCP<SaPFactory> SaPFact = rcp( new SaPFactory() );
SaPFact->SetDampingFactor(agg_damping);
PFact = SaPFact;
RFact = rcp( new TransPFactory() );
} else if (bEnergyMinimization == true) {
// Petrov Galerkin PG-AMG smoothed aggregation (energy minimization in ML)
PFact = rcp( new PgPFactory() );
RFact = rcp( new GenericRFactory() );
}
RCP<RAPFactory> AcFact = rcp( new RAPFactory() );
AcFact->SetParameter("RepairMainDiagonal", Teuchos::ParameterEntry(bFixDiagonal));
for (size_t i = 0; i<TransferFacts_.size(); i++) {
AcFact->AddTransferFactory(TransferFacts_[i]);
}
//
// introduce rebalancing
//
#if defined(HAVE_MUELU_ISORROPIA) && defined(HAVE_MPI)
Teuchos::RCP<Factory> RebalancedPFact = Teuchos::null;
Teuchos::RCP<Factory> RebalancedRFact = Teuchos::null;
Teuchos::RCP<Factory> RepartitionFact = Teuchos::null;
Teuchos::RCP<RebalanceAcFactory> RebalancedAFact = Teuchos::null;
MUELU_READ_PARAM(paramList, "repartition: enable", int, 0, bDoRepartition);
if (bDoRepartition == 1) {
// The Factory Manager will be configured to return the rebalanced versions of P, R, A by default.
// Everytime we want to use the non-rebalanced versions, we need to explicitly define the generating factory.
RFact->SetFactory("P", PFact);
//
AcFact->SetFactory("P", PFact);
AcFact->SetFactory("R", RFact);
MUELU_READ_PARAM(paramList, "repartition: max min ratio", double, 1.3, maxminratio);
MUELU_READ_PARAM(paramList, "repartition: min per proc", int, 512, minperproc);
// create "Partition"
Teuchos::RCP<MueLu::IsorropiaInterface<LO, GO, NO, LMO> > isoInterface = Teuchos::rcp(new MueLu::IsorropiaInterface<LO, GO, NO, LMO>());
isoInterface->SetFactory("A", AcFact);
// Repartitioning (creates "Importer" from "Partition")
RepartitionFact = Teuchos::rcp(new RepartitionFactory());
{
Teuchos::ParameterList paramListRepFact;
paramListRepFact.set("repartition: min rows per proc", minperproc);
paramListRepFact.set("repartition: max imbalance", maxminratio);
RepartitionFact->SetParameterList(paramListRepFact);
}
RepartitionFact->SetFactory("A", AcFact);
RepartitionFact->SetFactory("Partition", isoInterface);
// Reordering of the transfer operators
RebalancedPFact = Teuchos::rcp(new RebalanceTransferFactory());
RebalancedPFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Interpolation")));
RebalancedPFact->SetFactory("P", PFact);
RebalancedRFact = Teuchos::rcp(new RebalanceTransferFactory());
RebalancedRFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Restriction")));
RebalancedRFact->SetFactory("R", RFact);
RebalancedRFact->SetFactory("Nullspace", PtentFact);
// Compute Ac from rebalanced P and R
RebalancedAFact = Teuchos::rcp(new RebalanceAcFactory());
RebalancedAFact->SetFactory("A", AcFact);
}
void AdaptiveSaMLParameterListInterpreter<Scalar, LocalOrdinal, GlobalOrdinal, Node>::SetParameterList(const Teuchos::ParameterList & paramList_in) {
Teuchos::ParameterList paramList = paramList_in;
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); // TODO: use internal out (GetOStream())
//
// Read top-level of the parameter list
//
// hard-coded default values == ML defaults according to the manual
MUELU_READ_PARAM(paramList, "ML output", int, 0, verbosityLevel);
MUELU_READ_PARAM(paramList, "max levels", int, 10, maxLevels);
MUELU_READ_PARAM(paramList, "PDE equations", int, 1, nDofsPerNode);
MUELU_READ_PARAM(paramList, "coarse: max size", int, 128, maxCoarseSize);
MUELU_READ_PARAM(paramList, "aggregation: type", std::string, "Uncoupled", agg_type);
//MUELU_READ_PARAM(paramList, "aggregation: threshold", double, 0.0, agg_threshold);
MUELU_READ_PARAM(paramList, "aggregation: damping factor", double, (double)4/(double)3, agg_damping);
//MUELU_READ_PARAM(paramList, "aggregation: smoothing sweeps", int, 1, agg_smoothingsweeps);
MUELU_READ_PARAM(paramList, "aggregation: nodes per aggregate", int, 1, minPerAgg);
MUELU_READ_PARAM(paramList, "null space: type", std::string, "default vectors", nullspaceType);
MUELU_READ_PARAM(paramList, "null space: dimension", int, -1, nullspaceDim); // TODO: ML default not in documentation
MUELU_READ_PARAM(paramList, "null space: vectors", double*, NULL, nullspaceVec); // TODO: ML default not in documentation
MUELU_READ_PARAM(paramList, "energy minimization: enable", bool, false, bEnergyMinimization);
//
// Move smoothers/aggregation/coarse parameters to sublists
//
// ML allows to have level-specific smoothers/aggregation/coarse parameters at the top level of the list or/and defined in sublists:
// See also: ML Guide section 6.4.1, MueLu::CreateSublists, ML_CreateSublists
ParameterList paramListWithSubList;
MueLu::CreateSublists(paramList, paramListWithSubList);
paramList = paramListWithSubList; // swap
// std::cout << std::endl << "Parameter list after CreateSublists" << std::endl;
// std::cout << paramListWithSubList << std::endl;
int maxNbrAlreadySelected = 0;
// Matrix option
this->blksize_ = nDofsPerNode;
// Translate verbosity parameter
Teuchos::EVerbosityLevel eVerbLevel = Teuchos::VERB_NONE;
if (verbosityLevel == 0) eVerbLevel = Teuchos::VERB_NONE;
if (verbosityLevel > 0) eVerbLevel = Teuchos::VERB_LOW;
if (verbosityLevel > 4) eVerbLevel = Teuchos::VERB_MEDIUM;
if (verbosityLevel > 7) eVerbLevel = Teuchos::VERB_HIGH;
if (verbosityLevel > 9) eVerbLevel = Teuchos::VERB_EXTREME;
TEUCHOS_TEST_FOR_EXCEPTION(agg_type != "Uncoupled" && agg_type != "Coupled", Exceptions::RuntimeError, "MueLu::MLParameterListInterpreter::Setup(): parameter \"aggregation: type\": only 'Uncoupled' or 'Coupled' aggregation is supported.");
// Create MueLu factories
// RCP<NullspaceFactory> nspFact = rcp(new NullspaceFactory());
RCP<CoalesceDropFactory> dropFact = rcp(new CoalesceDropFactory());
//dropFact->SetVerbLevel(toMueLuVerbLevel(eVerbLevel));
RCP<FactoryBase> CoupledAggFact = Teuchos::null;
if(agg_type == "Uncoupled") {
// Uncoupled aggregation
RCP<UncoupledAggregationFactory> CoupledAggFact2 = rcp(new UncoupledAggregationFactory());
CoupledAggFact2->SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D
CoupledAggFact2->SetMaxNeighAlreadySelected(maxNbrAlreadySelected);
CoupledAggFact2->SetOrdering("natural");
CoupledAggFact = CoupledAggFact2;
} else {
// Coupled Aggregation (default)
RCP<CoupledAggregationFactory> CoupledAggFact2 = rcp(new CoupledAggregationFactory());
CoupledAggFact2->SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D
CoupledAggFact2->SetMaxNeighAlreadySelected(maxNbrAlreadySelected);
CoupledAggFact2->SetOrdering("natural");
CoupledAggFact2->SetPhase3AggCreation(0.5);
CoupledAggFact = CoupledAggFact2;
}
if (verbosityLevel > 3) { // TODO fix me: Setup is a static function: we cannot use GetOStream without an object...
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min Nodes per aggregate : " << minPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << maxNbrAlreadySelected << std::endl;
*out << "aggregate ordering : natural" << std::endl;
*out << "=============================================================================" << std::endl;
}
RCP<Factory> PFact;
RCP<Factory> RFact;
RCP<Factory> PtentFact = rcp( new TentativePFactory() );
if (agg_damping == 0.0 && bEnergyMinimization == false) {
// tentative prolongation operator (PA-AMG)
PFact = PtentFact;
RFact = rcp( new TransPFactory() );
} else if (agg_damping != 0.0 && bEnergyMinimization == false) {
// smoothed aggregation (SA-AMG)
RCP<SaPFactory> SaPFact = rcp( new SaPFactory() );
SaPFact->SetParameter("sa: damping factor", ParameterEntry(agg_damping));
PFact = SaPFact;
RFact = rcp( new TransPFactory() );
} else if (bEnergyMinimization == true) {
// Petrov Galerkin PG-AMG smoothed aggregation (energy minimization in ML)
PFact = rcp( new PgPFactory() );
RFact = rcp( new GenericRFactory() );
}
RCP<RAPFactory> AcFact = rcp( new RAPFactory() );
for (size_t i = 0; i<TransferFacts_.size(); i++) {
AcFact->AddTransferFactory(TransferFacts_[i]); // THIS WILL BE REPLACED with a call to the MLParamterListInterpreter
}
//
// Nullspace factory
//
// Set fine level nullspace
// extract pre-computed nullspace from ML parameter list
// store it in nullspace_ and nullspaceDim_
if (nullspaceType != "default vectors") {
TEUCHOS_TEST_FOR_EXCEPTION(nullspaceType != "pre-computed", Exceptions::RuntimeError, "MueLu::MLParameterListInterpreter: no valid nullspace (no pre-computed null space). error.");
TEUCHOS_TEST_FOR_EXCEPTION(nullspaceDim == -1, Exceptions::RuntimeError, "MueLu::MLParameterListInterpreter: no valid nullspace (nullspace dim == -1). error.");
TEUCHOS_TEST_FOR_EXCEPTION(nullspaceVec == NULL, Exceptions::RuntimeError, "MueLu::MLParameterListInterpreter: no valid nullspace (nullspace == NULL). You have to provide a valid fine-level nullspace in \'null space: vectors\'");
nullspaceDim_ = nullspaceDim;
nullspace_ = nullspaceVec;
}
Teuchos::RCP<NullspaceFactory> nspFact = Teuchos::rcp(new NullspaceFactory());
nspFact->SetFactory("Nullspace", PtentFact);
//
// Hierarchy + FactoryManager
//
// Hierarchy options
this->SetVerbLevel(toMueLuVerbLevel(eVerbLevel));
this->numDesiredLevel_ = maxLevels;
this->maxCoarseSize_ = maxCoarseSize;
// init smoother
RCP<SmootherFactory> initSmootherFact = Teuchos::null;
if(paramList.isSublist("init smoother")) {
ParameterList& initList = paramList.sublist("init smoother"); // TODO move this before for loop
initSmootherFact = MLParameterListInterpreter::GetSmootherFactory(initList); // TODO: missing AFact input arg.
} else {
std::string ifpackType = "RELAXATION";
Teuchos::ParameterList smootherParamList;
smootherParamList.set("relaxation: type", "symmetric Gauss-Seidel");
smootherParamList.set("smoother: sweeps", 1);
smootherParamList.set("smoother: damping factor", 1.0);
RCP<SmootherPrototype> smooProto = rcp( new TrilinosSmoother(ifpackType, smootherParamList, 0) );
initSmootherFact = rcp( new SmootherFactory() );
initSmootherFact->SetSmootherPrototypes(smooProto, smooProto);
}
//
// Coarse Smoother
//
ParameterList& coarseList = paramList.sublist("coarse: list");
// coarseList.get("smoother: type", "Amesos-KLU"); // set default
//RCP<SmootherFactory> coarseFact = this->GetSmootherFactory(coarseList);
RCP<SmootherFactory> coarseFact = MLParameterListInterpreter::GetSmootherFactory(coarseList);
// Smoothers Top Level Parameters
RCP<ParameterList> topLevelSmootherParam = ExtractSetOfParameters(paramList, "smoother");
// std::cout << std::endl << "Top level smoother parameters:" << std::endl;
// std::cout << *topLevelSmootherParam << std::endl;
//
// Prepare factory managers
// TODO: smootherFact can be reuse accross level if same parameters/no specific parameterList
for (int levelID=0; levelID < maxLevels; levelID++) {
//
// Level FactoryManager
//
RCP<FactoryManager> manager = rcp(new FactoryManager());
RCP<FactoryManager> initmanager = rcp(new FactoryManager());
//
// Smoothers
//
{
// Merge level-specific parameters with global parameters. level-specific parameters takes precedence.
// TODO: unit-test this part alone
ParameterList levelSmootherParam = GetMLSubList(paramList, "smoother", levelID); // copy
MergeParameterList(*topLevelSmootherParam, levelSmootherParam, false); /* false = do no overwrite levelSmootherParam parameters by topLevelSmootherParam parameters */
// std::cout << std::endl << "Merged List for level " << levelID << std::endl;
// std::cout << levelSmootherParam << std::endl;
//RCP<SmootherFactory> smootherFact = this->GetSmootherFactory(levelSmootherParam); // TODO: missing AFact input arg.
RCP<SmootherFactory> smootherFact = MLParameterListInterpreter::GetSmootherFactory(levelSmootherParam); // TODO: missing AFact input arg.
manager->SetFactory("Smoother", smootherFact);
smootherFact->DisableMultipleCallCheck();
initmanager->SetFactory("Smoother", initSmootherFact);
initmanager->SetFactory("CoarseSolver", initSmootherFact);
initSmootherFact->DisableMultipleCallCheck();
}
//
// Misc
//
Teuchos::rcp_dynamic_cast<PFactory>(PFact)->DisableMultipleCallCheck();
Teuchos::rcp_dynamic_cast<PFactory>(PtentFact)->DisableMultipleCallCheck();
Teuchos::rcp_dynamic_cast<TwoLevelFactoryBase>(RFact)->DisableMultipleCallCheck();
Teuchos::rcp_dynamic_cast<SingleLevelFactoryBase>(coarseFact)->DisableMultipleCallCheck();
Teuchos::rcp_dynamic_cast<SingleLevelFactoryBase>(dropFact)->DisableMultipleCallCheck();
Teuchos::rcp_dynamic_cast<SingleLevelFactoryBase>(CoupledAggFact)->DisableMultipleCallCheck();
Teuchos::rcp_dynamic_cast<TwoLevelFactoryBase>(AcFact)->DisableMultipleCallCheck();
Teuchos::rcp_dynamic_cast<SingleLevelFactoryBase>(nspFact)->DisableMultipleCallCheck();
manager->SetFactory("CoarseSolver", coarseFact); // TODO: should not be done in the loop
manager->SetFactory("Graph", dropFact);
manager->SetFactory("Aggregates", CoupledAggFact);
manager->SetFactory("DofsPerNode", dropFact);
manager->SetFactory("A", AcFact);
manager->SetFactory("P", PFact);
manager->SetFactory("Ptent", PtentFact);
manager->SetFactory("R", RFact);
manager->SetFactory("Nullspace", nspFact);
//initmanager->SetFactory("CoarseSolver", coarseFact);
initmanager->SetFactory("Graph", dropFact);
initmanager->SetFactory("Aggregates", CoupledAggFact);
initmanager->SetFactory("DofsPerNode", dropFact);
initmanager->SetFactory("A", AcFact);
initmanager->SetFactory("P", PtentFact); // use nonsmoothed transfers
initmanager->SetFactory("Ptent", PtentFact);
initmanager->SetFactory("R", RFact);
initmanager->SetFactory("Nullspace", nspFact);
this->AddFactoryManager(levelID, 1, manager);
this->AddInitFactoryManager(levelID, 1, initmanager);
} // for (level loop)
}
TEUCHOS_UNIT_TEST(MultiVectorTransferFactory, ThreeLevels)
{
out << "version: " << MueLu::Version() << std::endl;
out << "Tests usage on a three-level hierarchy." << std::endl;
GO nx = 199;
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::Build1DPoisson(nx);
// Set up three level hierarchy.
RCP<Hierarchy> H = rcp( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
RCP<Level> fineLevel = H->GetLevel();
fineLevel->setDefaultVerbLevel(Teuchos::VERB_HIGH);
fineLevel->Set("A",A); // set fine level matrix
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(A->getRowMap(),1);
nullSpace->putScalar( (SC) 1.0);
fineLevel->Set("Nullspace",nullSpace); // set null space information for finest level
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetMinNodesPerAggregate(3);
CoupledAggFact->SetMaxNeighAlreadySelected(0);
CoupledAggFact->SetOrdering(MueLu::AggOptions::NATURAL);
CoupledAggFact->SetPhase3AggCreation(0.5);
RCP<TentativePFactory> PFact = rcp(new TentativePFactory()); //just using plain aggregation
RCP<Factory> RFact = rcp(new TransPFactory());
RCP<RAPFactory> AcFact = rcp(new RAPFactory());
H->SetMaxCoarseSize(1);
Teuchos::ParameterList smootherParamList;
smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
smootherParamList.set("relaxation: sweeps", (LO) 1);
smootherParamList.set("relaxation: damping factor", (SC) 1.0);
RCP<SmootherPrototype> smooProto = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );
RCP<SmootherFactory> SmooFact = rcp( new SmootherFactory(smooProto) );
AcFact->setVerbLevel(Teuchos::VERB_HIGH);
FactoryManager M;
M.SetFactory("Aggregates", CoupledAggFact);
M.SetFactory("P", PFact);
M.SetFactory("Ptent", PFact); // for nullspace
M.SetFactory("R", RFact);
M.SetFactory("A", AcFact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", SmooFact); // This line avoid dependency to Amesos/Amesos2 for this test.
//set up the transfer factory
RCP<MultiVector> fineOnes = MultiVectorFactory::Build(A->getRowMap(),1);
fineOnes->putScalar(1.0);
fineLevel->Set("onesVector",fineOnes);
RCP<MueLu::MultiVectorTransferFactory<SC, LO, GO, NO, LMO> > mvtf = rcp(new MueLu::MultiVectorTransferFactory<SC, LO, GO, NO, LMO>("onesVector"));
mvtf->SetFactory("R",RFact);
M.SetFactory("onesVector",mvtf);
AcFact->AddTransferFactory(mvtf);
int maxLevels = 3;
H->Setup(M, 0, maxLevels);
/*
//FIXME we probably need to do some requests....
coarseLevel.Request("onesVector",mvtf.get());
coarseLevel.Request("R",RFact.get());
coarseLevel.Request("P",TentativePFact.get());
*/
/*
RCP<MultiVector> coarseOnes = coarseLevel.Get<RCP<MultiVector> >("onesVector",mvtf.get());
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> vn(1);
coarseOnes->norm2(vn);
TEST_FLOATING_EQUALITY(vn[0]*vn[0],((SC)fineOnes->getGlobalLength()),1e-12);
*/
} // ThreeLevels