void ReuseTpetraPreconditioner(const Teuchos::RCP<Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inA,
MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node>& Op) {
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Xpetra::Operator<SC,LO,GO,NO> Operator;
typedef MueLu ::Hierarchy<SC,LO,GO,NO> Hierarchy;
RCP<Hierarchy> H = Op.GetHierarchy();
TEUCHOS_TEST_FOR_EXCEPTION(!H->GetNumLevels(), Exceptions::RuntimeError,
"ReuseTpetraPreconditioner: Hierarchy has no levels in it");
TEUCHOS_TEST_FOR_EXCEPTION(!H->GetLevel(0)->IsAvailable("A"), Exceptions::RuntimeError,
"ReuseTpetraPreconditioner: Hierarchy has no fine level operator");
RCP<Level> level0 = H->GetLevel(0);
RCP<Operator> O0 = level0->Get<RCP<Operator> >("A");
RCP<Matrix> A0 = Teuchos::rcp_dynamic_cast<Matrix>(O0);
RCP<Matrix> A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(inA);
if (!A0.is_null()) {
// If a user provided a "number of equations" argument in a parameter list
// during the initial setup, we must honor that settings and reuse it for
// all consequent setups.
A->SetFixedBlockSize(A0->GetFixedBlockSize());
}
level0->Set("A", A);
H->SetupRe();
}
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(ParameterListInterpreter, SetParameterList, Scalar, LocalOrdinal, GlobalOrdinal, Node)
{
# include <MueLu_UseShortNames.hpp>
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_SCOPE(Scalar,GlobalOrdinal,Node);
#if defined(HAVE_MUELU_TPETRA) && defined(HAVE_MUELU_EPETRA) && defined(HAVE_MUELU_IFPACK) && defined(HAVE_MUELU_IFPACK2) && defined(HAVE_MUELU_AMESOS) && defined(HAVE_MUELU_AMESOS2)
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO>::Build1DPoisson(99);
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
ArrayRCP<std::string> fileList = TestHelpers::GetFileList(std::string("ParameterList/ParameterListInterpreter/"), std::string(".xml"));
for(int i=0; i< fileList.size(); i++) {
out << "Processing file: " << fileList[i] << std::endl;
ParameterListInterpreter mueluFactory("ParameterList/ParameterListInterpreter/" + fileList[i],*comm);
RCP<Hierarchy> H = mueluFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
mueluFactory.SetupHierarchy(*H);
//TODO: check no unused parameters
//TODO: check results of Iterate()
}
# else
out << "Skipping test because some required packages are not enabled (Tpetra, Epetra, EpetraExt, Ifpack, Ifpack2, Amesos, Amesos2)." << std::endl;
# endif
}
TEUCHOS_UNIT_TEST(MLParameterListInterpreter, SetParameterList)
{
//TODO: this test can be done at compilation time
#if !defined(HAVE_MUELU_EPETRA) or !defined(HAVE_MUELU_IFPACK) or !defined(HAVE_MUELU_AMESOS)
if (TestHelpers::Parameters::getLib() == Xpetra::UseEpetra) {
out << "Test skipped (dependencies not available)" << std::endl;
return;
}
#endif
#if !defined(HAVE_MUELU_TPETRA) or !defined(HAVE_MUELU_IFPACK2) or !defined(HAVE_MUELU_AMESOS2)
if (TestHelpers::Parameters::getLib() == Xpetra::UseTpetra) {
out << "Test skipped (dependencies not available)" << std::endl;
return;
}
#endif
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO>::Build1DPoisson(99);
Teuchos::ParameterList galeriParameters;
galeriParameters.set("nx",99);
RCP<MultiVector> coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", A->getRowMap(), galeriParameters);
ArrayRCP<std::string> fileList = TestHelpers::GetFileList(std::string("ParameterList/MLParameterListInterpreter/"), std::string(".xml"));
for(int i=0; i< fileList.size(); i++) {
out << "Processing file: " << fileList[i] << std::endl;
Teuchos::ParameterList myList;
myList.set("xml parameter file","ParameterList/MLParameterListInterpreter/" + fileList[i]);
Teuchos::ArrayRCP<MultiVector::scalar_type> xcoord=coordinates->getDataNonConst(0);
myList.set("x-coordinates",xcoord.get());
MLParameterListInterpreter mueluFactory(myList,A->getRowMap()->getComm());
RCP<Hierarchy> H = mueluFactory.CreateHierarchy();
H->GetLevel(0)->Set<RCP<Matrix> >("A", A);
mueluFactory.SetupHierarchy(*H);
//TODO: check no unused parameters
//TODO: check results of Iterate()
}
}
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
/**********************************************************************************/
/* SET TEST PARAMETERS */
/**********************************************************************************/
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561=3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748=4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// custom parameters
int nSmoothers=2;
LO maxLevels = 3;
LO its=10;
std::string coarseSolver="ifpack2";
// std::string coarseSolver="amesos2";
int pauseForDebugger=0;
clp.setOption("nSmoothers",&nSmoothers,"number of Gauss-Seidel smoothers in the MergedSmootehrs");
clp.setOption("maxLevels",&maxLevels,"maximum number of levels allowed. If 1, then a MergedSmoother is used on the coarse grid");
clp.setOption("its",&its,"number of multigrid cycles");
clp.setOption("coarseSolver",&coarseSolver,"amesos2 or ifpack2 (Tpetra specific. Ignored for Epetra)");
clp.setOption("debug",&pauseForDebugger,"pause to attach debugger");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
if (comm->getRank() == 0) {
// matrixParameters.print();
// xpetraParameters.print();
// TODO: print custom parameters
}
if (pauseForDebugger) {
Utils::PauseForDebugger();
}
/**********************************************************************************/
/* CREATE INITIAL MATRIX */
/**********************************************************************************/
const RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
RCP<Matrix> Op = Pr->BuildMatrix();
#ifdef NEUMANN
// Tranform matrix to Neumann b.c.
// Essentially, we need to update two diagonal elements
// TODO: calls to getLocalRowView not really needed
Op->resumeFill();
Teuchos::ArrayView<const LO> indices;
Teuchos::ArrayView<const SC> values;
Teuchos::Array<SC> newValues(2, 0.0);
size_t myRank = Op->getRowMap()->getComm()->getRank();
size_t nCpus = Op->getRowMap()->getComm()->getSize();
if (myRank == 0) { // JG TODO: can we use rowMap->isNodeLocalElement(0) instead for more genericity?
//LO firstRow = 0;
newValues[0] = 1.0; newValues[1] = -1.0;
Op->getLocalRowView(0, indices, values);
Op->replaceLocalValues(0, indices, newValues);
}
if (myRank == nCpus-1) { // JG TODO: can we use rowMap->isNodeLocalElement(lastRow) instead for more genericity?
LO lastRow = Op->getNodeNumRows()-1;
newValues[0] = -1.0; newValues[1] = 1.0;
Op->getLocalRowView(lastRow, indices, values);
Op->replaceLocalValues(lastRow, indices, newValues);
}
Op->fillComplete();
#endif // NEUMANN
/**********************************************************************************/
/* */
/**********************************************************************************/
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
std::cout << "||NS|| = " << norms[0] << std::endl;
RCP<MueLu::Hierarchy<SC,LO,GO,NO,LMO> > H = rcp( new Hierarchy() );
H->SetDefaultVerbLevel(MueLu::Extreme);
H->IsPreconditioner(false);
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A", Op);
Finest->Set("Nullspace", nullSpace);
FactoryManager M;
M.SetFactory("Aggregates", rcp(new CoupledAggregationFactory()));
M.SetFactory("Ptent", rcp(new TentativePFactory()));
M.SetFactory("P", rcp(new SaPFactory()));
#ifdef EMIN
// Energy-minimization
RCP<PatternFactory> PatternFact = rcp(new PatternFactory());
#if 0
PatternFact->SetFactory("P", M.GetFactory("Ptent"));
#else
PatternFact->SetFactory("P", M.GetFactory("P"));
#endif
M.SetFactory("Ppattern", PatternFact);
RCP<EminPFactory> EminPFact = rcp(new EminPFactory());
EminPFact->SetFactory("P", M.GetFactory("Ptent"));
M.SetFactory("P", EminPFact);
RCP<NullspacePresmoothFactory> NullPreFact = rcp(new NullspacePresmoothFactory());
NullPreFact->SetFactory("Nullspace", M.GetFactory("Nullspace"));
M.SetFactory("Nullspace", NullPreFact);
#endif
RCP<SmootherPrototype> smooProto = gimmeMergedSmoother(nSmoothers, xpetraParameters.GetLib(), coarseSolver, comm->getRank());
M.SetFactory("Smoother", rcp(new SmootherFactory(smooProto)));
Teuchos::ParameterList status;
RCP<SmootherPrototype> coarseProto;
if (maxLevels != 1)
coarseProto = gimmeCoarseProto(xpetraParameters.GetLib(), coarseSolver, comm->getRank());
else
coarseProto = gimmeMergedSmoother(nSmoothers, xpetraParameters.GetLib(), coarseSolver, comm->getRank());
if (coarseProto == Teuchos::null)
return EXIT_FAILURE;
#ifdef NEUMANN
// Use coarse level projection solver
RCP<SmootherPrototype> projectedSolver = rcp(new ProjectorSmoother(coarseProto));
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(projectedSolver));
#else
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(coarseProto));
#endif
M.SetFactory("CoarseSolver", coarseSolveFact);
H->EnableGraphDumping("graph.dot", 2);
H->Setup(M, 0, maxLevels);
//if (comm->getRank() == 0) {
// std::cout << "======================\n Multigrid statistics \n======================" << std::endl;
// status.print(std::cout,Teuchos::ParameterList::PrintOptions().indent(2));
//}
// Define RHS
RCP<MultiVector> X = MultiVectorFactory::Build(map,1);
RCP<MultiVector> RHS = MultiVectorFactory::Build(map,1);
X->setSeed(846930886);
X->randomize();
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_true|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
Op->apply(*X,*RHS,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
// Use AMG directly as an iterative method
{
X->putScalar( (SC) 0.0);
H->Iterate(*RHS,its,*X);
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_" << std::setprecision(2) << its << "|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
}
return EXIT_SUCCESS;
}
int main_(Teuchos::CommandLineProcessor &clp, int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int numProc = comm->getSize();
int myRank = comm->getRank();
// =========================================================================
// Parameters initialization
// =========================================================================
::Xpetra::Parameters xpetraParameters(clp);
bool runHeavyTests = false;
clp.setOption("heavytests", "noheavytests", &runHeavyTests, "whether to exercise tests that take a long time to run");
clp.recogniseAllOptions(true);
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
// =========================================================================
// Problem construction
// =========================================================================
ParameterList matrixParameters;
matrixParameters.set("nx", Teuchos::as<GO>(9999));
matrixParameters.set("matrixType", "Laplace1D");
RCP<Matrix> A = MueLuTests::TestHelpers::TestFactory<SC, LO, GO, NO>::Build1DPoisson(matrixParameters.get<GO>("nx"), lib);
RCP<MultiVector> coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", A->getRowMap(), matrixParameters);
std::string outDir = "Output/";
std::vector<std::string> dirList;
if (runHeavyTests) {
dirList.push_back("EasyParameterListInterpreter-heavy/");
dirList.push_back("FactoryParameterListInterpreter-heavy/");
} else {
dirList.push_back("EasyParameterListInterpreter/");
dirList.push_back("FactoryParameterListInterpreter/");
}
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ISORROPIA) && defined(HAVE_AMESOS2_KLU2)
// The ML interpreter have internal ifdef, which means that the resulting
// output would depend on configuration (reguarl interpreter does not have
// that). Therefore, we need to stabilize the configuration here.
// In addition, we run ML parameter list tests only if KLU is available
dirList.push_back("MLParameterListInterpreter/");
dirList.push_back("MLParameterListInterpreter2/");
#endif
int numLists = dirList.size();
bool failed = false;
Teuchos::Time timer("Interpreter timer");
//double lastTime = timer.wallTime();
for (int k = 0; k < numLists; k++) {
Teuchos::ArrayRCP<std::string> fileList = MueLuTests::TestHelpers::GetFileList(dirList[k],
(numProc == 1 ? std::string(".xml") : std::string("_np" + Teuchos::toString(numProc) + ".xml")));
for (int i = 0; i < fileList.size(); i++) {
// Set seed
std::srand(12345);
// Reset (potentially) cached value of the estimate
A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());
std::string xmlFile = dirList[k] + fileList[i];
std::string outFile = outDir + fileList[i];
std::string baseFile = outFile.substr(0, outFile.find_last_of('.'));
std::size_t found = baseFile.find("_np");
if (numProc == 1 && found != std::string::npos) {
#ifdef HAVE_MPI
baseFile = baseFile.substr(0, found);
#else
std::cout << "Skipping \"" << xmlFile << "\" as MPI is not enabled" << std::endl;
continue;
#endif
}
baseFile = baseFile + (lib == Xpetra::UseEpetra ? "_epetra" : "_tpetra");
std::string goldFile = baseFile + ".gold";
std::ifstream f(goldFile.c_str());
if (!f.good()) {
if (myRank == 0)
std::cout << "Warning: comparison file " << goldFile << " not found. Skipping test" << std::endl;
continue;
}
std::filebuf buffer;
std::streambuf* oldbuffer = NULL;
if (myRank == 0) {
// Redirect output
buffer.open((baseFile + ".out").c_str(), std::ios::out);
oldbuffer = std::cout.rdbuf(&buffer);
}
// NOTE: we cannot use ParameterListInterpreter(xmlFile, comm), because we want to update the ParameterList
// first to include "test" verbosity
Teuchos::ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFile, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
if (dirList[k] == "EasyParameterListInterpreter/" || dirList[k] == "EasyParameterListInterpreter-heavy/")
paramList.set("verbosity", "test");
else if (dirList[k] == "FactoryParameterListInterpreter/" || dirList[k] == "FactoryParameterListInterpreter-heavy/")
paramList.sublist("Hierarchy").set("verbosity", "Test");
else if (dirList[k] == "MLParameterListInterpreter/")
paramList.set("ML output", 42);
else if (dirList[k] == "MLParameterListInterpreter2/")
paramList.set("ML output", 10);
try {
timer.start();
Teuchos::RCP<HierarchyManager> mueluFactory;
// create parameter list interpreter
// here we have to distinguish between the general MueLu parameter list interpreter
// and the ML parameter list interpreter. Note that the ML paramter interpreter also
// works with Tpetra matrices.
if (dirList[k] == "EasyParameterListInterpreter/" ||
dirList[k] == "EasyParameterListInterpreter-heavy/" ||
dirList[k] == "FactoryParameterListInterpreter/" ||
dirList[k] == "FactoryParameterListInterpreter-heavy/") {
mueluFactory = Teuchos::rcp(new ParameterListInterpreter(paramList));
} else if (dirList[k] == "MLParameterListInterpreter/") {
mueluFactory = Teuchos::rcp(new MLParameterListInterpreter(paramList));
} else if (dirList[k] == "MLParameterListInterpreter2/") {
//std::cout << "ML ParameterList: " << std::endl;
//std::cout << paramList << std::endl;
RCP<ParameterList> mueluParamList = Teuchos::getParametersFromXmlString(MueLu::ML2MueLuParameterTranslator::translate(paramList,"SA"));
//std::cout << "MueLu ParameterList: " << std::endl;
//std::cout << *mueluParamList << std::endl;
mueluFactory = Teuchos::rcp(new ParameterListInterpreter(*mueluParamList));
}
RCP<Hierarchy> H = mueluFactory->CreateHierarchy();
H->GetLevel(0)->template Set<RCP<Matrix> >("A", A);
if (dirList[k] == "MLParameterListInterpreter/") {
// MLParameterInterpreter needs the nullspace information if rebalancing is active!
// add default constant null space vector
RCP<MultiVector> nullspace = MultiVectorFactory::Build(A->getRowMap(), 1);
nullspace->putScalar(1.0);
H->GetLevel(0)->Set("Nullspace", nullspace);
}
H->GetLevel(0)->Set("Coordinates", coordinates);
mueluFactory->SetupHierarchy(*H);
if (strncmp(fileList[i].c_str(), "reuse", 5) == 0) {
// Build the Hierarchy the second time
// Should be faster if we actually do the reuse
A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());
mueluFactory->SetupHierarchy(*H);
}
timer.stop();
} catch (Teuchos::ExceptionBase& e) {
std::string msg = e.what();
msg = msg.substr(msg.find_last_of('\n')+1);
if (myRank == 0) {
std::cout << "Caught exception: " << msg << std::endl;
// Redirect output back
std::cout.rdbuf(oldbuffer);
buffer.close();
}
if (msg == "Zoltan interface is not available" ||
msg == "Zoltan2 interface is not available" ||
msg == "MueLu::FactoryFactory:BuildFactory(): Cannot create a Zoltan2Interface object: Zoltan2 is disabled: HAVE_MUELU_ZOLTAN2 && HAVE_MPI == false.") {
if (myRank == 0)
std::cout << xmlFile << ": skipped (missing library)" << std::endl;
continue;
}
}
std::string cmd;
if (myRank == 0) {
// Redirect output back
std::cout.rdbuf(oldbuffer);
buffer.close();
// Create a copy of outputs
cmd = "cp -f ";
system((cmd + baseFile + ".gold " + baseFile + ".gold_filtered").c_str());
system((cmd + baseFile + ".out " + baseFile + ".out_filtered").c_str());
// Tpetra produces different eigenvalues in Chebyshev due to using
// std::rand() for generating random vectors, which may be initialized
// using different seed, and may have different algorithm from one
// gcc version to another, or to anogther compiler (like clang)
// This leads to us always failing this test.
// NOTE1 : Epetra, on the other hand, rolls out its out random number
// generator, which always produces same results
// Ignore the value of "lambdaMax"
run_sed("'s/lambdaMax: [0-9]*.[0-9]*/lambdaMax = <ignored>/'", baseFile);
// Ignore the value of "lambdaMin"
run_sed("'s/lambdaMin: [0-9]*.[0-9]*/lambdaMin = <ignored>/'", baseFile);
// Ignore the value of "chebyshev: max eigenvalue"
// NOTE: we skip lines with default value ([default])
run_sed("'/[default]/! s/chebyshev: max eigenvalue = [0-9]*.[0-9]*/chebyshev: max eigenvalue = <ignored>/'", baseFile);
// Ignore the exact type of direct solver (it is selected semi-automatically
// depending on how Trilinos was configured
run_sed("'s/Amesos\\([2]*\\)Smoother{type = .*}/Amesos\\1Smoother{type = <ignored>}/'", baseFile);
run_sed("'s/SuperLU solver interface, direct solve/<Direct> solver interface/'", baseFile);
run_sed("'s/KLU2 solver interface/<Direct> solver interface/'", baseFile);
run_sed("'s/Basker solver interface/<Direct> solver interface/'", baseFile);
// Strip template args for some classes
std::vector<std::string> classes;
classes.push_back("Xpetra::Matrix");
classes.push_back("MueLu::Constraint");
classes.push_back("MueLu::SmootherPrototype");
for (size_t q = 0; q < classes.size(); q++)
run_sed("'s/" + classes[q] + "<.*>/" + classes[q] + "<ignored> >/'", baseFile);
#ifdef __APPLE__
// Some Macs print outs ptrs as 0x0 instead of 0, fix that
run_sed("'/RCP/ s/=0x0/=0/g'", baseFile);
#endif
// Run comparison (ignoring whitespaces)
cmd = "diff -u -w -I\"^\\s*$\" " + baseFile + ".gold_filtered " + baseFile + ".out_filtered";
int ret = system(cmd.c_str());
if (ret)
failed = true;
//std::ios_base::fmtflags ff(std::cout.flags());
//std::cout.precision(2);
//std::cout << xmlFile << " (" << std::setiosflags(std::ios::fixed)
// << timer.wallTime() - lastTime << " sec.) : " << (ret ? "failed" : "passed") << std::endl;
//lastTime = timer.wallTime();
//std::cout.flags(ff); // reset flags to whatever they were prior to printing time
std::cout << xmlFile << " : " << (ret ? "failed" : "passed") << std::endl;
}
}
}
if (myRank == 0)
std::cout << std::endl << "End Result: TEST " << (failed ? "FAILED" : "PASSED") << std::endl;
return (failed ? EXIT_FAILURE : EXIT_SUCCESS);
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
//TODO: FIXME: option by default does not work for MueLu/Tpetra
int nIts = 9;
Teuchos::CommandLineProcessor clp(false); // Note:
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 256); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
std::string xmlFileName; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default an hard-coded parameter list.");
int muelu = true; clp.setOption("muelu", &muelu, "use muelu"); //TODO: bool instead of int
int ml = true;
#if defined(HAVE_MUELU_ML) && defined(HAVE_MUELU_EPETRA)
clp.setOption("ml", &ml, "use ml");
#endif
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
// TODO: check -ml and --linAlgebra
if (comm->getRank() == 0) { std::cout << xpetraParameters << matrixParameters; }
if (ml && xpetraParameters.GetLib() == Xpetra::UseTpetra) {
ml = false;
std::cout << "ML preconditionner can only be built if --linAlgebra=Epetra. Option --ml ignored" << std::endl;
}
//
// Construct the problem
//
RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Map, CrsMatrixWrap, MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
//
// Preconditionner configuration
//
// ML parameter list
RCP<Teuchos::ParameterList> params;
if (xmlFileName != "") {
std::cout << "Reading " << xmlFileName << " ..." << std::endl;
params = Teuchos::getParametersFromXmlFile(xmlFileName);
} else {
std::cout << "Using hard-coded parameter list:" << std::endl;
params = rcp(new Teuchos::ParameterList());
params->set("ML output", 10);
params->set("max levels", 2);
params->set("smoother: type", "symmetric Gauss-Seidel");
if (xpetraParameters.GetLib() == Xpetra::UseTpetra) // TODO: remove 'if' when Amesos2-KLU becomes available
params->set("coarse: type","Amesos-Superlu");
else
params->set("coarse: type","Amesos-KLU");
}
std::cout << "Initial parameter list" << std::endl;
std::cout << *params << std::endl;
if (muelu) {
//
// Construct a multigrid preconditioner
//
// Multigrid Hierarchy
MLParameterListInterpreter mueLuFactory(*params);
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
// build default null space
LocalOrdinal numPDEs = 1;
if(A->IsView("stridedMaps")==true) {
Xpetra::viewLabel_t oldView = A->SwitchToView("stridedMaps"); // note: "stridedMaps are always non-overlapping (correspond to range and domain maps!)
numPDEs = Teuchos::rcp_dynamic_cast<const StridedMap>(A->getRowMap())->getFixedBlockSize();
oldView = A->SwitchToView(oldView);
}
RCP<MultiVector> nullspace = MultiVectorFactory::Build(A->getDomainMap(), numPDEs);
for (int i=0; i<numPDEs; ++i) {
Teuchos::ArrayRCP<Scalar> nsValues = nullspace->getDataNonConst(i);
int numBlocks = nsValues.size() / numPDEs;
for (int j=0; j< numBlocks; ++j) {
nsValues[j*numPDEs + i] = 1.0;
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("A", A);
//
// build hierarchy
//
mueLuFactory.SetupHierarchy(*H);
//
// Solve Ax = b
//
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
X->putScalar((Scalar) 0.0);
B->setSeed(846930886); B->randomize();
// AMG as a standalone solver
H->IsPreconditioner(false);
H->Iterate(*B, *X, nIts);
// Print relative residual norm
Teuchos::ScalarTraits<SC>::magnitudeType residualNorms = Utils::ResidualNorm(*A, *X, *B)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
#if defined(HAVE_MUELU_EPETRA) && defined(HAVE_MUELU_AZTECOO)
if (xpetraParameters.GetLib() == Xpetra::UseEpetra) { //TODO: should be doable with Tpetra too
// AMG as a preconditioner
//TODO: name mueluPrec and mlPrec not
H->IsPreconditioner(true);
MueLu::EpetraOperator mueluPrec(H); // Wrap MueLu preconditioner into an Epetra Operator
//
// Solve Ax = b
//
RCP<Epetra_CrsMatrix> eA; //duplicate code
{ // TODO: simplify this
RCP<CrsMatrixWrap> xCrsOp = Teuchos::rcp_dynamic_cast<CrsMatrixWrap>(A, true);
RCP<CrsMatrix> xCrsMtx = xCrsOp->getCrsMatrix();
RCP<EpetraCrsMatrix> eCrsMtx = Teuchos::rcp_dynamic_cast<EpetraCrsMatrix>(xCrsMtx, true);
eA = eCrsMtx->getEpetra_CrsMatrixNonConst();
}
RCP<Epetra_Vector> eX = rcp(new Epetra_Vector(eA->RowMap()));
RCP<Epetra_Vector> eB = rcp(new Epetra_Vector(eA->RowMap()));
eX->PutScalar((Scalar) 0.0);
eB->SetSeed(846930886); eB->Random();
Epetra_LinearProblem eProblem(eA.get(), eX.get(), eB.get());
// AMG as a standalone solver
AztecOO solver(eProblem);
solver.SetPrecOperator(&mueluPrec);
solver.SetAztecOption(AZ_solver, AZ_fixed_pt);
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(nIts, 1e-10);
{ //TODO: simplify this
RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(eX));
RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(eB));
// Print relative residual norm
Teuchos::ScalarTraits<SC>::magnitudeType residualNorms2 = Utils::ResidualNorm(*A, *mueluX, *mueluB)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms2 << std::endl;
}
// TODO: AMG as a preconditioner (AZ_cg)
}
#endif // HAVE_MUELU_AZTECOO
} // if (muelu)
#if defined(HAVE_MUELU_ML) && defined(HAVE_MUELU_EPETRA)
if (ml) {
std::cout << std::endl << std::endl << std::endl << std::endl << "**** ML ml ML ml ML" << std::endl << std::endl << std::endl << std::endl;
//
// Construct a multigrid preconditioner
//
// Multigrid Hierarchy
RCP<CrsMatrixWrap> crsOp = Teuchos::rcp_dynamic_cast<CrsMatrixWrap>(A, true);
RCP<CrsMatrix> crsMtx = crsOp->getCrsMatrix();
RCP<EpetraCrsMatrix> epetraCrsMtx = Teuchos::rcp_dynamic_cast<EpetraCrsMatrix>(crsMtx, true);
RCP<const Epetra_CrsMatrix> epetra_CrsMtx = epetraCrsMtx->getEpetra_CrsMatrix();
RCP<Epetra_CrsMatrix> eA;
{ // TODO: simplify this
RCP<CrsMatrixWrap> xCrsOp = Teuchos::rcp_dynamic_cast<CrsMatrixWrap>(A, true);
RCP<CrsMatrix> xCrsMtx = xCrsOp->getCrsMatrix();
RCP<EpetraCrsMatrix> eCrsMtx = Teuchos::rcp_dynamic_cast<EpetraCrsMatrix>(xCrsMtx, true);
eA = eCrsMtx->getEpetra_CrsMatrixNonConst();
}
RCP<ML_Epetra::MultiLevelPreconditioner> mlPrec = rcp(new ML_Epetra::MultiLevelPreconditioner(*eA, *params));
#ifdef HAVE_MUELU_AZTECOO
//
// Solve Ax = b
//
RCP<Epetra_Vector> eX = rcp(new Epetra_Vector(eA->RowMap()));
RCP<Epetra_Vector> eB = rcp(new Epetra_Vector(eA->RowMap()));
eX->PutScalar((Scalar) 0.0);
eB->SetSeed(846930886); eB->Random();
Epetra_LinearProblem eProblem(eA.get(), eX.get(), eB.get());
// AMG as a standalone solver
AztecOO solver(eProblem);
solver.SetPrecOperator(mlPrec.get());
solver.SetAztecOption(AZ_solver, AZ_fixed_pt);
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(nIts, 1e-10);
{ //TODO: simplify this
RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(eX));
RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(eB));
// Print relative residual norm
Teuchos::ScalarTraits<SC>::magnitudeType residualNorms = Utils::ResidualNorm(*A, *mueluX, *mueluB)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
}
// TODO: AMG as a preconditioner (AZ_cg)
#else
std::cout << "Enable AztecOO to see solution" << std::endl;
#endif // HAVE_MUELU_AZTECOO
std::cout << "Parameter list after ML run" << std::endl;
const Teuchos::ParameterList & paramsAfterML = mlPrec->GetList();
std::cout << paramsAfterML << std::endl;
} // if (ml)
#endif // HAVE_MUELU_ML && HAVE_MUELU_EPETRA
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP;
using Teuchos::rcp;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
bool success = false;
try {
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
#endif
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
std::string xmlFileName = "xml/muelu_ParameterList.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file [default = 'xml/muelu_ParameterList.xml']");
int globalNumDofs = 0; //7020;
clp.setOption("globalNumDofs", &globalNumDofs, "global number of degrees of freedom [has to be set by user, default = 0 -> error]");
int nDofsPerNode = 1;
clp.setOption("nDofsPerNode", &nDofsPerNode, "number of degrees of freedom per node [has to be set by user, default = 1]");
int nProcs = comm->getSize();
std::string dsolveType = "cg";
clp.setOption("solver", &dsolveType, "solve type: (none | cg | gmres | standalone) [default = cg]");
double dtol = 1e-12;
clp.setOption("tol", &dtol, "solver convergence tolerance [default = 1e-12]");
std::string problemFile = "stru2d";
clp.setOption("problem", &problemFile, "string for problem file (e.g. 'stru2d' expects 'stru2d_A.txt', 'stru2d_b.txt' and 'stru2d_ns.txt')");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if(globalNumDofs == 0) {
std::cout << "Please specify '--globalNumDofs'! Simulation cannot run without that parameter correctly set" << std::endl;
return EXIT_FAILURE;
}
int nLocalDofs = (int) globalNumDofs / nProcs;
nLocalDofs = nLocalDofs - (nLocalDofs % nDofsPerNode);
int nCumulatedDofs = 0;
sumAll(comm,nLocalDofs, nCumulatedDofs);
if(comm->getRank() == nProcs-1) {
nLocalDofs += globalNumDofs - nCumulatedDofs;
}
// read in problem
Epetra_Map emap (globalNumDofs, nLocalDofs, 0, *Xpetra::toEpetra(comm));
Epetra_CrsMatrix * ptrA = 0;
Epetra_Vector * ptrf = 0;
Epetra_MultiVector* ptrNS = 0;
std::cout << "Reading matrix market file" << std::endl;
std::stringstream ssA, ssB, ssNS;
ssA << problemFile << "_A.txt";
ssB << problemFile << "_b.txt";
ssNS << problemFile << "_ns.txt";
std::string fileA = ssA.str();
std::string fileB = ssB.str();
std::string fileNS = ssNS.str();
EpetraExt::MatrixMarketFileToCrsMatrix(fileA.c_str(),emap,emap,emap,ptrA);
EpetraExt::MatrixMarketFileToVector(fileB.c_str(),emap,ptrf);
EpetraExt::MatrixMarketFileToMultiVector(fileNS.c_str(), emap, ptrNS);
RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
RCP<Epetra_Vector> epB = Teuchos::rcp(ptrf);
RCP<Epetra_MultiVector> epNS = Teuchos::rcp(ptrNS);
// Epetra_CrsMatrix -> Xpetra::Matrix
RCP<CrsMatrix> exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(epA));
RCP<CrsMatrixWrap> crsOp = Teuchos::rcp(new CrsMatrixWrap(exA));
RCP<Matrix> Op = Teuchos::rcp_dynamic_cast<Matrix>(crsOp);
Op->SetFixedBlockSize(nDofsPerNode);
RCP<MultiVector> xNS = Teuchos::rcp(new Xpetra::EpetraMultiVector(epNS));
// Epetra_Map -> Xpetra::Map
const RCP< const Map> map = Xpetra::toXpetra<GO>(emap);
ParameterListInterpreter mueLuFactory(xmlFileName,*comm);
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
RCP<MueLu::Level> Finest = H->GetLevel(0);
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op);
Finest->Set("Nullspace",xNS);
mueLuFactory.SetupHierarchy(*H);
#ifdef HAVE_MUELU_AZTECOO
H->IsPreconditioner(true);
MueLu::EpetraOperator mueluPrec(H); // Wrap MueLu preconditioner into an Epetra Operator
// create a solution vector
RCP<Epetra_Vector> epX = rcp(new Epetra_Vector(epA->RowMap()));
epX->PutScalar((Scalar) 0.0);
Epetra_LinearProblem eProblem(epA.get(), epX.get(), epB.get());
// AMG as preconditioner within AztecOO
AztecOO solver(eProblem);
solver.SetPrecOperator(&mueluPrec);
if (dsolveType == "cg")
solver.SetAztecOption(AZ_solver, AZ_cg);
else if (dsolveType == "gmres")
solver.SetAztecOption(AZ_solver, AZ_gmres);
else { // use fix point method instead
solver.SetAztecOption(AZ_solver, AZ_fixed_pt);
}
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(500, dtol);
{ //TODO: simplify this
RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(epX));
RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(epB));
// Print relative residual norm
Teuchos::ScalarTraits<SC>::magnitudeType residualNorms = Utils::ResidualNorm(*Op, *mueluX, *mueluB)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
}
#endif // HAVE_MUELU_AZTECOO
success = true;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
void MueLuPreconditionerFactory::initializePrec(
const Teuchos::RCP<const LinearOpSourceBase<double> > &fwdOpSrc,
PreconditionerBase<double> *prec,
const ESupportSolveUse supportSolveUse
) const
{
using Teuchos::outArg;
using Teuchos::OSTab;
using Teuchos::dyn_cast;
using Teuchos::RCP;
using Teuchos::null;
using Teuchos::rcp;
using Teuchos::rcp_dynamic_cast;
using Teuchos::rcp_const_cast;
using Teuchos::set_extra_data;
using Teuchos::get_optional_extra_data;
using Teuchos::implicit_cast;
typedef KokkosClassic::DefaultNode::DefaultNodeType NO;
typedef KokkosClassic::DefaultKernels<double,int,NO>::SparseOps LMO;
Teuchos::Time totalTimer(""), timer("");
totalTimer.start(true);
const RCP<Teuchos::FancyOStream> out = this->getOStream();
const Teuchos::EVerbosityLevel verbLevel = this->getVerbLevel();
Teuchos::OSTab tab(out);
if(out.get() && implicit_cast<int>(verbLevel) > implicit_cast<int>(Teuchos::VERB_LOW))
*out << "\nEntering Thyra::MueLuPreconditionerFactory::initializePrec(...) ...\n";
Teuchos::RCP<const LinearOpBase<double> > fwdOp = fwdOpSrc->getOp();
#ifdef _DEBUG
TEUCHOS_TEST_FOR_EXCEPT(fwdOp.get()==NULL);
TEUCHOS_TEST_FOR_EXCEPT(prec==NULL);
#endif
//
// Unwrap and get the forward Epetra_Operator object
//
Teuchos::RCP<const Epetra_Operator> epetraFwdOp;
EOpTransp epetraFwdOpTransp;
EApplyEpetraOpAs epetraFwdOpApplyAs;
EAdjointEpetraOp epetraFwdOpAdjointSupport;
double epetraFwdOpScalar;
epetraFwdOpViewExtractor_->getEpetraOpView(
fwdOp,outArg(epetraFwdOp),outArg(epetraFwdOpTransp),outArg(epetraFwdOpApplyAs),
outArg(epetraFwdOpAdjointSupport),outArg(epetraFwdOpScalar)
);
// Validate what we get is what we need
RCP<const Epetra_CrsMatrix>
epetraFwdCrsMat = rcp_dynamic_cast<const Epetra_CrsMatrix>(epetraFwdOp,true);
TEUCHOS_TEST_FOR_EXCEPTION(
epetraFwdOpApplyAs != EPETRA_OP_APPLY_APPLY, std::logic_error
,"Error, incorrect apply mode for an Epetra_CrsMatrix"
);
//
// Get the concrete preconditioner object
//
DefaultPreconditioner<double>
*defaultPrec = &Teuchos::dyn_cast<DefaultPreconditioner<double> >(*prec);
//
// Get the EpetraLinearOp object that is used to implement the preconditoner linear op
//
RCP<EpetraLinearOp>
epetra_precOp = rcp_dynamic_cast<EpetraLinearOp>(defaultPrec->getNonconstUnspecifiedPrecOp(),true);
//
// Get the embedded MueLu::EpetraOperator object if it exists
//
Teuchos::RCP<MueLu::EpetraOperator> muelu_precOp;
if(epetra_precOp.get())
muelu_precOp = rcp_dynamic_cast<MueLu::EpetraOperator>(epetra_precOp->epetra_op(),true);
//
// Get the attached forward operator if it exists and make sure that it matches
//
if(muelu_precOp!=Teuchos::null) {
// TODO
// // Get the forward operator and make sure that it matches what is
// // already being used!
// const Epetra_CrsMatrix & rm = muelu_precOp->CrsMatrix();
// TEUCHOS_TEST_FOR_EXCEPTION(
// &rm!=&*epetraFwdRowMat, std::logic_error
// ,"MueLu requires Epetra_RowMatrix to be the same for each initialization of the preconditioner"
// );
}
MueLu::ParameterListInterpreter<double, int, int, NO, LMO> mueluFactory(*paramList_);
//
// Perform initialization if needed
//
const bool startingOver = (muelu_precOp.get() == NULL);
if(startingOver)
{
if(out.get() && implicit_cast<int>(verbLevel) >= implicit_cast<int>(Teuchos::VERB_LOW))
*out << "\nCreating the initial MueLu::EpetraOperator object...\n";
timer.start(true);
// Create the initial preconditioner: DO NOT compute it yet
// Turns a Epetra_CrsMatrix into a Xpetra::Matrix
RCP<Epetra_CrsMatrix> epetraFwdCrsMatNonConst = rcp_const_cast<Epetra_CrsMatrix>(epetraFwdCrsMat); // !! TODO: MueLu interface should accept const matrix as input.
RCP<Xpetra::CrsMatrix<double, int, int, NO, LMO> > mueluAcrs = rcp(new Xpetra::EpetraCrsMatrix(epetraFwdCrsMatNonConst)); //TODO: should not be needed
RCP<Xpetra::Matrix <double, int, int, NO, LMO> > mueluA = rcp(new Xpetra::CrsMatrixWrap<double, int, int, NO, LMO>(mueluAcrs));
const RCP<MueLu::Hierarchy<double,int, int, NO, LMO > > muelu_hierarchy = mueluFactory.CreateHierarchy();
muelu_hierarchy->GetLevel(0)->Set("A", mueluA);
muelu_precOp = rcp(new MueLu::EpetraOperator(muelu_hierarchy));
timer.stop();
if(out.get() && implicit_cast<int>(verbLevel) >= implicit_cast<int>(Teuchos::VERB_LOW))
OSTab(out).o() <<"> Creation time = "<<timer.totalElapsedTime()<<" sec\n";
// if(paramList_.get())
// TEUCHOS_TEST_FOR_EXCEPT(
// 0!=muelu_precOp->SetParameterList(paramList_->sublist(MueLuSettings_name))
// );
}
//
// Attach the epetraFwdOp to the muelu_precOp to guarantee that it will not go away
//
set_extra_data(epetraFwdOp, "IFPF::epetraFwdOp", Teuchos::inOutArg(muelu_precOp),
Teuchos::POST_DESTROY, false);
//
// Update the factorization
//
if(out.get() && implicit_cast<int>(verbLevel) >= implicit_cast<int>(Teuchos::VERB_LOW))
*out << "\nComputing the preconditioner ...\n";
timer.start(true);
mueluFactory.SetupHierarchy(*muelu_precOp->GetHierarchy());
timer.stop();
if(out.get() && implicit_cast<int>(verbLevel) >= implicit_cast<int>(Teuchos::VERB_LOW))
OSTab(out).o() <<"=> Setup time = "<<timer.totalElapsedTime()<<" sec\n";
//
// Compute the conditioner number estimate if asked
//
// ToDo: Implement
//
// Attach fwdOp to the muelu_precOp
//
set_extra_data(fwdOp, "IFPF::fwdOp", Teuchos::inOutArg(muelu_precOp),
Teuchos::POST_DESTROY, false);
//
// Initialize the output EpetraLinearOp
//
if(startingOver) {
epetra_precOp = rcp(new EpetraLinearOp);
}
epetra_precOp->initialize(
muelu_precOp
,epetraFwdOpTransp
,EPETRA_OP_APPLY_APPLY_INVERSE
,EPETRA_OP_ADJOINT_UNSUPPORTED // ToDo: Look into adjoints again.
);
//
// Initialize the preconditioner
//
defaultPrec->initializeUnspecified(
Teuchos::rcp_implicit_cast<LinearOpBase<double> >(epetra_precOp)
);
totalTimer.stop();
if(out.get() && implicit_cast<int>(verbLevel) >= implicit_cast<int>(Teuchos::VERB_LOW))
*out << "\nTotal time in MLPreconditionerFactory = "<<totalTimer.totalElapsedTime()<<" sec\n";
if(out.get() && implicit_cast<int>(verbLevel) > implicit_cast<int>(Teuchos::VERB_LOW))
*out << "\nLeaving Thyra::MLPreconditionerFactory::initializePrec(...) ...\n";
}
int main(int argc, char *argv[]) {
#if defined(HAVE_MUELU_EPETRA)
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
bool success = false;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int MyPID = comm->getRank();
int NumProc = comm->getSize();
const Teuchos::RCP<Epetra_Comm> epComm = Teuchos::rcp_const_cast<Epetra_Comm>(Xpetra::toEpetra(comm));
// =========================================================================
// Convenient definitions
// =========================================================================
//SC zero = Teuchos::ScalarTraits<SC>::zero(), one = Teuchos::ScalarTraits<SC>::one();
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100;
GO maxCoarseSize = 10;
LO maxLevels = 4;
clp.setOption("nx", &nx, "mesh size in x direction");
clp.setOption("ny", &ny, "mesh size in y direction");
clp.setOption("maxCoarseSize", &maxCoarseSize, "maximum coarse size");
clp.setOption("maxLevels", &maxLevels, "maximum number of multigrid levels");
int mgridSweeps = 1; clp.setOption("mgridSweeps", &mgridSweeps, "number of multigrid sweeps within Multigrid solver.");
std::string printTimings = "no"; clp.setOption("timings", &printTimings, "print timings to screen [yes/no]");
double tol = 1e-12; clp.setOption("tol", &tol, "solver convergence tolerance");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
// =========================================================================
// Problem construction
// =========================================================================
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time"))), tm;
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
Teuchos::ParameterList GaleriList;
GaleriList.set("nx", nx);
GaleriList.set("ny", ny);
GaleriList.set("mx", epComm->NumProc());
GaleriList.set("my", 1);
GaleriList.set("lx", 1.0); // length of x-axis
GaleriList.set("ly", 1.0); // length of y-axis
GaleriList.set("diff", 1e-5);
GaleriList.set("conv", 1.0);
// create map
Teuchos::RCP<Epetra_Map> epMap = Teuchos::rcp(Galeri::CreateMap("Cartesian2D", *epComm, GaleriList));
// create coordinates
Teuchos::RCP<Epetra_MultiVector> epCoord = Teuchos::rcp(Galeri::CreateCartesianCoordinates("2D", epMap.get(), GaleriList));
// create matrix
Teuchos::RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(Galeri::CreateCrsMatrix("Recirc2D", epMap.get(), GaleriList));
// Epetra -> Xpetra
Teuchos::RCP<CrsMatrix> exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrixT<int,Node>(epA));
Teuchos::RCP<CrsMatrixWrap> exAWrap = Teuchos::rcp(new CrsMatrixWrap(exA));
RCP<Matrix> A = Teuchos::rcp_dynamic_cast<Matrix>(exAWrap);
int numPDEs = 1;
A->SetFixedBlockSize(numPDEs);
// set rhs and solution vector
RCP<Epetra_Vector> B = Teuchos::rcp(new Epetra_Vector(*epMap));
RCP<Epetra_Vector> X = Teuchos::rcp(new Epetra_Vector(*epMap));
B->PutScalar(1.0);
X->PutScalar(0.0);
// Epetra -> Xpetra
RCP<Vector> xB = Teuchos::rcp(new Xpetra::EpetraVectorT<int,Node>(B));
RCP<Vector> xX = Teuchos::rcp(new Xpetra::EpetraVectorT<int,Node>(X));
xX->setSeed(100);
xX->randomize();
// build null space vector
RCP<const Map> map = A->getRowMap();
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, numPDEs);
for (int i=0; i<numPDEs; ++i) {
Teuchos::ArrayRCP<Scalar> nsValues = nullspace->getDataNonConst(i);
int numBlocks = nsValues.size() / numPDEs;
for (int j=0; j< numBlocks; ++j) {
nsValues[j*numPDEs + i] = 1.0;
}
}
comm->barrier();
tm = Teuchos::null;
fancyout << "========================================================\nGaleri complete.\n========================================================" << std::endl;
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
RCP<Hierarchy> H = rcp ( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
H->SetMaxCoarseSize(maxCoarseSize);
// build finest Level
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",A);
Finest->Set("Nullspace",nullspace);
// create factories for transfer operators
RCP<TentativePFactory> PFact = Teuchos::rcp(new TentativePFactory());
RCP<TransPFactory> RFact = Teuchos::rcp(new TransPFactory());
RFact->SetFactory("P", PFact);
// build level smoothers
// use symmetric Gauss-Seidel both for fine and coarse level smoother
RCP<SmootherPrototype> smooProto;
std::string ifpackType;
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) 1);
ifpackList.set("relaxation: damping factor", (SC) 1.0);
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
smooProto = Teuchos::rcp( new TrilinosSmoother(ifpackType, ifpackList) );
RCP<SmootherFactory> SmooFact;
if (maxLevels > 1)
SmooFact = rcp( new SmootherFactory(smooProto) );
// design multigrid hierarchy
FactoryManager M;
M.SetFactory("P", PFact);
M.SetFactory("R", RFact);
M.SetFactory("Nullspace", PFact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", SmooFact);
H->Setup(M, 0, maxLevels);
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
//
// generate exact solution using a direct solver
//
RCP<Epetra_Vector> exactLsgVec = rcp(new Epetra_Vector(X->Map()));
{
fancyout << "========================================================\nCalculate exact solution." << std::endl;
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - direct solve")));
exactLsgVec->PutScalar(0.0);
exactLsgVec->Update(1.0,*X,1.0);
Epetra_LinearProblem epetraProblem(epA.get(), exactLsgVec.get(), B.get());
Amesos amesosFactory;
RCP<Amesos_BaseSolver> rcp_directSolver = Teuchos::rcp(amesosFactory.Create("Amesos_Klu", epetraProblem));
rcp_directSolver->SymbolicFactorization();
rcp_directSolver->NumericFactorization();
rcp_directSolver->Solve();
comm->barrier();
tm = Teuchos::null;
}
//
// Solve Ax = b using AMG as a preconditioner in AztecOO
//
RCP<Epetra_Vector> precLsgVec = rcp(new Epetra_Vector(X->Map()));
{
fancyout << "========================================================\nUse multigrid hierarchy as preconditioner within CG." << std::endl;
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - AMG as preconditioner")));
precLsgVec->PutScalar(0.0);
precLsgVec->Update(1.0,*X,1.0);
Epetra_LinearProblem epetraProblem(epA.get(), precLsgVec.get(), B.get());
AztecOO aztecSolver(epetraProblem);
aztecSolver.SetAztecOption(AZ_solver, AZ_gmres);
MueLu::EpetraOperator aztecPrec(H);
aztecSolver.SetPrecOperator(&aztecPrec);
int maxIts = 100;
//double tol2 = 1e-8;
aztecSolver.Iterate(maxIts, tol);
comm->barrier();
tm = Teuchos::null;
}
//////////////////
// use multigrid hierarchy as solver
RCP<Vector> mgridLsgVec = VectorFactory::Build(map);
mgridLsgVec->putScalar(0.0);
{
fancyout << "========================================================\nUse multigrid hierarchy as solver." << std::endl;
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Multigrid Solve")));
mgridLsgVec->update(1.0,*xX,1.0);
H->IsPreconditioner(false);
H->Iterate(*xB, *mgridLsgVec, mgridSweeps);
comm->barrier();
tm = Teuchos::null;
}
//////////////////
fancyout << "========================================================\nExport results.\n========================================================" << std::endl;
std::ofstream myfile;
std::stringstream ss; ss << "example" << MyPID << ".txt";
myfile.open (ss.str().c_str());
//////////////////
// loop over all procs
for (int iproc=0; iproc < NumProc; iproc++) {
if (MyPID==iproc) {
int NumVectors1 = 2;
int NumMyElements1 = epCoord->Map(). NumMyElements();
int MaxElementSize1 = epCoord->Map().MaxElementSize();
int * FirstPointInElementList1 = NULL;
if (MaxElementSize1!=1) FirstPointInElementList1 = epCoord->Map().FirstPointInElementList();
double ** A_Pointers = epCoord->Pointers();
if (MyPID==0) {
myfile.width(8);
myfile << "# MyPID"; myfile << " ";
myfile.width(12);
if (MaxElementSize1==1)
myfile << "GID ";
else
myfile << " GID/Point";
for (int j = 0; j < NumVectors1 ; j++)
{
myfile.width(20);
myfile << "Value ";
}
myfile << std::endl;
}
for (int i=0; i < NumMyElements1; i++) {
for (int ii=0; ii< epCoord->Map().ElementSize(i); ii++) {
int iii;
myfile.width(10);
myfile << MyPID; myfile << " ";
myfile.width(10);
if (MaxElementSize1==1) {
if(epCoord->Map().GlobalIndicesInt())
{
int * MyGlobalElements1 = epCoord->Map().MyGlobalElements();
myfile << MyGlobalElements1[i] << " ";
}
iii = i;
}
else {
if(epCoord->Map().GlobalIndicesInt())
{
int * MyGlobalElements1 = epCoord->Map().MyGlobalElements();
myfile << MyGlobalElements1[i]<< "/" << ii << " ";
}
iii = FirstPointInElementList1[i]+ii;
}
for (int j = 0; j < NumVectors1 ; j++)
{
myfile.width(20);
myfile << A_Pointers[j][iii];
}
myfile.precision(18); // set high precision for output
// add solution vector entry
myfile.width(25); myfile << (*exactLsgVec)[iii];
// add preconditioned solution vector entry
myfile.width(25); myfile << (*precLsgVec)[iii];
Teuchos::ArrayRCP<SC> mgridLsgVecData = mgridLsgVec->getDataNonConst(0);
myfile.width(25); myfile << mgridLsgVecData[iii];
myfile.precision(6); // set default precision
myfile << std::endl;
}
} // end loop over all lines on current proc
myfile << std::flush;
// syncronize procs
comm->barrier();
comm->barrier();
comm->barrier();
} // end myProc
}
////////////
myfile.close();
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings == "yes") {
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union, "", true);
}
success = true;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
#else
return EXIT_SUCCESS;
#endif // #if defined(HAVE_MUELU_EPETRA) and defined(HAVE_MUELU_SERIAL)
} //main
int main(int argc, char *argv[]) {
#if defined(HAVE_MUELU_EPETRA) && defined(HAVE_MUELU_EPETRAEXT)
typedef double Scalar;
typedef int LocalOrdinal;
typedef int GlobalOrdinal;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Xpetra::EpetraNode Node;
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP;
using Teuchos::rcp;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
bool success = false;
bool verbose = true;
try {
//
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// Timing
Teuchos::Time myTime("global");
Teuchos::TimeMonitor MM(myTime);
// custom parameters
LocalOrdinal maxLevels = 3;
GlobalOrdinal maxCoarseSize=1; //FIXME clp doesn't like long long int
int globalNumDofs = 1500; // used for the maps
int nDofsPerNode = 3; // used for generating the fine level null-space
// build strided maps
// striding information: 2 velocity dofs and 1 pressure dof = 3 dofs per node
std::vector<size_t> stridingInfo;
stridingInfo.push_back(2);
stridingInfo.push_back(1);
/////////////////////////////////////// build strided maps
// build strided maps:
// xstridedfullmap: full map (velocity and pressure dof gids), continous
// xstridedvelmap: only velocity dof gid maps (i.e. 0,1,3,4,6,7...)
// xstridedpremap: only pressure dof gid maps (i.e. 2,5,8,...)
Xpetra::UnderlyingLib lib = Xpetra::UseEpetra;
RCP<StridedMap> xstridedfullmap = StridedMapFactory::Build(lib,globalNumDofs,0,stridingInfo,comm,-1);
RCP<StridedMap> xstridedvelmap = StridedMapFactory::Build(xstridedfullmap,0);
RCP<StridedMap> xstridedpremap = StridedMapFactory::Build(xstridedfullmap,1);
/////////////////////////////////////// transform Xpetra::Map objects to Epetra
// this is needed for AztecOO
const RCP<const Epetra_Map> fullmap = Teuchos::rcpFromRef(Xpetra::toEpetra(*xstridedfullmap));
RCP<const Epetra_Map> velmap = Teuchos::rcpFromRef(Xpetra::toEpetra(*xstridedvelmap));
RCP<const Epetra_Map> premap = Teuchos::rcpFromRef(Xpetra::toEpetra(*xstridedpremap));
/////////////////////////////////////// import problem matrix and RHS from files (-> Epetra)
// read in problem
Epetra_CrsMatrix * ptrA = 0;
Epetra_Vector * ptrf = 0;
Epetra_MultiVector* ptrNS = 0;
*out << "Reading matrix market file" << std::endl;
EpetraExt::MatrixMarketFileToCrsMatrix("A_re1000_5932.txt",*fullmap,*fullmap,*fullmap,ptrA);
EpetraExt::MatrixMarketFileToVector("b_re1000_5932.txt",*fullmap,ptrf);
RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
RCP<Epetra_Vector> epv = Teuchos::rcp(ptrf);
RCP<Epetra_MultiVector> epNS = Teuchos::rcp(ptrNS);
/////////////////////////////////////// split system into 2x2 block system
*out << "Split matrix into 2x2 block matrix" << std::endl;
// split fullA into A11,..., A22
Teuchos::RCP<Epetra_CrsMatrix> A11;
Teuchos::RCP<Epetra_CrsMatrix> A12;
Teuchos::RCP<Epetra_CrsMatrix> A21;
Teuchos::RCP<Epetra_CrsMatrix> A22;
if(MueLuTests::SplitMatrix2x2(epA,*velmap,*premap,A11,A12,A21,A22)==false)
*out << "Problem with splitting matrix"<< std::endl;
/////////////////////////////////////// transform Epetra objects to Xpetra (needed for MueLu)
// build Xpetra objects from Epetra_CrsMatrix objects
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA11 = Teuchos::rcp(new Xpetra::EpetraCrsMatrixT<GlobalOrdinal,Node>(A11));
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA12 = Teuchos::rcp(new Xpetra::EpetraCrsMatrixT<GlobalOrdinal,Node>(A12));
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA21 = Teuchos::rcp(new Xpetra::EpetraCrsMatrixT<GlobalOrdinal,Node>(A21));
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA22 = Teuchos::rcp(new Xpetra::EpetraCrsMatrixT<GlobalOrdinal,Node>(A22));
/////////////////////////////////////// generate MapExtractor object
std::vector<Teuchos::RCP<const Xpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > > xmaps;
xmaps.push_back(xstridedvelmap);
xmaps.push_back(xstridedpremap);
Teuchos::RCP<const Xpetra::MapExtractor<Scalar,LocalOrdinal,GlobalOrdinal,Node> > map_extractor = Xpetra::MapExtractorFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node>::Build(xstridedfullmap,xmaps);
/////////////////////////////////////// build blocked transfer operator
// using the map extractor
Teuchos::RCP<Xpetra::BlockedCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > bOp = Teuchos::rcp(new Xpetra::BlockedCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>(map_extractor,map_extractor,10));
bOp->setMatrix(0,0,xA11);
bOp->setMatrix(0,1,xA12);
bOp->setMatrix(1,0,xA21);
bOp->setMatrix(1,1,xA22);
bOp->fillComplete();
//////////////////////////////////////////////////// create Hierarchy
RCP<Hierarchy> H = rcp ( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
//H->setDefaultVerbLevel(Teuchos::VERB_NONE);
H->SetMaxCoarseSize(maxCoarseSize);
//////////////////////////////////////////////////////// finest Level
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Teuchos::rcp_dynamic_cast<Matrix>(bOp));
/////////////////////////////////////////////// define subblocks of A
// make A11 block and A22 block available as variable "A" generated
// by A11Fact and A22Fact
RCP<SubBlockAFactory> A11Fact = rcp(new SubBlockAFactory());
A11Fact->SetFactory("A",MueLu::NoFactory::getRCP());
A11Fact->SetParameter("block row",Teuchos::ParameterEntry(0));
A11Fact->SetParameter("block col",Teuchos::ParameterEntry(0));
RCP<SubBlockAFactory> A22Fact = rcp(new SubBlockAFactory());
A22Fact->SetFactory("A",MueLu::NoFactory::getRCP());
A22Fact->SetParameter("block row",Teuchos::ParameterEntry(1));
A22Fact->SetParameter("block col",Teuchos::ParameterEntry(1));
////////////////////////////////////////// prepare null space for A11
RCP<MultiVector> nullspace11 = MultiVectorFactory::Build(xstridedvelmap, 2); // this is a 2D standard null space
for (int i=0; i<nDofsPerNode-1; ++i) {
Teuchos::ArrayRCP<Scalar> nsValues = nullspace11->getDataNonConst(i);
int numBlocks = nsValues.size() / (nDofsPerNode - 1);
for (int j=0; j< numBlocks; ++j) {
nsValues[j*(nDofsPerNode - 1) + i] = 1.0;
}
}
Finest->Set("Nullspace1",nullspace11);
///////////////////////////////////////// define CoalesceDropFactory and Aggregation for A11
// set up amalgamation for A11. Note: we're using a default null space factory (Teuchos::null)
RCP<AmalgamationFactory> amalgFact11 = rcp(new AmalgamationFactory());
amalgFact11->SetFactory("A", A11Fact);
amalgFact11->setDefaultVerbLevel(Teuchos::VERB_EXTREME);
RCP<CoalesceDropFactory> dropFact11 = rcp(new CoalesceDropFactory());
dropFact11->SetFactory("A", A11Fact);
dropFact11->SetFactory("UnAmalgamationInfo", amalgFact11);
dropFact11->setDefaultVerbLevel(Teuchos::VERB_EXTREME);
RCP<UncoupledAggregationFactory> CoupledAggFact11 = rcp(new UncoupledAggregationFactory());
CoupledAggFact11->SetFactory("Graph", dropFact11);
CoupledAggFact11->SetMinNodesPerAggregate(9);
CoupledAggFact11->SetMaxNeighAlreadySelected(2);
CoupledAggFact11->SetOrdering("natural");
//CoupledAggFact11->SetPhase3AggCreation(0.5);
///////////////////////////////////////// define transfer ops for A11
#if 0
// use PG-AMG
RCP<PgPFactory> P11Fact = rcp(new PgPFactory());
RCP<GenericRFactory> R11Fact = rcp(new GenericRFactory());
Teuchos::RCP<NullspaceFactory> nspFact11 = Teuchos::rcp(new NullspaceFactory("Nullspace1",P11tentFact));
Teuchos::RCP<NullspaceFactory> nspFact11 = Teuchos::rcp(new NullspaceFactory("Nullspace1"));
RCP<CoarseMapFactory> coarseMapFact11 = Teuchos::rcp(new CoarseMapFactory());
coarseMapFact11->setStridingData(stridingInfo);
coarseMapFact11->setStridedBlockId(0);
//////////////////////////////// define factory manager for (1,1) block
RCP<FactoryManager> M11 = rcp(new FactoryManager());
M11->SetFactory("A", A11Fact);
M11->SetFactory("P", P11Fact);
M11->SetFactory("R", R11Fact);
M11->SetFactory("Aggregates", CoupledAggFact11);
M11->SetFactory("UnAmalgamationInfo", amalgFact11);
M11->SetFactory("Nullspace", nspFact11);
// M11->SetFactory("Ptent", P11tentFact);
M11->SetFactory("CoarseMap", coarseMapFact11);
#else
RCP<TentativePFactory> P11Fact = rcp(new TentativePFactory());
RCP<TransPFactory> R11Fact = rcp(new TransPFactory());
Teuchos::RCP<NullspaceFactory> nspFact11 = Teuchos::rcp(new NullspaceFactory("Nullspace1"));
nspFact11->SetFactory("Nullspace1",P11Fact);
RCP<CoarseMapFactory> coarseMapFact11 = Teuchos::rcp(new CoarseMapFactory());
coarseMapFact11->setStridingData(stridingInfo);
coarseMapFact11->setStridedBlockId(0);
//////////////////////////////// define factory manager for (1,1) block
RCP<FactoryManager> M11 = rcp(new FactoryManager());
M11->SetFactory("A", A11Fact);
M11->SetFactory("P", P11Fact);
M11->SetFactory("R", R11Fact);
M11->SetFactory("Aggregates", CoupledAggFact11);
M11->SetFactory("UnAmalgamationInfo", amalgFact11);
M11->SetFactory("Nullspace", nspFact11);
// M11->SetFactory("Ptent", P11Fact);
M11->SetFactory("CoarseMap", coarseMapFact11);
#endif
M11->SetIgnoreUserData(true); // always use data from factories defined in factory manager
////////////////////////////////////////// prepare null space for A22
RCP<MultiVector> nullspace22 = MultiVectorFactory::Build(xstridedpremap, 1); // this is a 2D standard null space
Teuchos::ArrayRCP<Scalar> nsValues22 = nullspace22->getDataNonConst(0);
for (int j=0; j< nsValues22.size(); ++j) {
nsValues22[j] = 1.0;
}
Finest->Set("Nullspace2",nullspace22);
///////////////////////////////////////// define transfer ops for A22
#if 0
// use PGAMG
RCP<AmalgamationFactory> amalgFact22 = rcp(new AmalgamationFactory(A22Fact));
RCP<TentativePFactory> P22tentFact = rcp(new TentativePFactory(CoupledAggFact11, amalgFact22));
RCP<SaPFactory> P22Fact = rcp(new SaPFactory(P22tentFact));
//RCP<GenericRFactory> R22Fact = rcp(new GenericRFactory(P22Fact));
RCP<TransPFactory> R22Fact = rcp(new TransPFactory(P22Fact));
Teuchos::RCP<NullspaceFactory> nspFact22 = Teuchos::rcp(new NullspaceFactory("Nullspace2",P22tentFact));
RCP<CoarseMapFactory> coarseMapFact22 = Teuchos::rcp(new CoarseMapFactory(CoupledAggFact11, nspFact22));
coarseMapFact22->setStridingData(stridingInfo);
coarseMapFact22->setStridedBlockId(1);
//////////////////////////////// define factory manager for (2,2) block
RCP<FactoryManager> M22 = rcp(new FactoryManager());
M22->SetFactory("A", A22Fact);
M22->SetFactory("P", P22Fact);
M22->SetFactory("R", R22Fact);
M22->SetFactory("Aggregates", AggFact22);
M22->SetFactory("Nullspace", nspFact22);
M22->SetFactory("Ptent", P22tentFact);
M22->SetFactory("CoarseMap", coarseMapFact22);
M22->SetIgnoreUserData(true); // always use data from factories defined in factory manager
#else
// use TentativePFactory
RCP<AmalgamationFactory> amalgFact22 = rcp(new AmalgamationFactory());
RCP<TentativePFactory> P22Fact = rcp(new TentativePFactory()); // check me (fed with A22) wrong column GIDS!!!
RCP<TransPFactory> R22Fact = rcp(new TransPFactory());
Teuchos::RCP<NullspaceFactory> nspFact22 = Teuchos::rcp(new NullspaceFactory("Nullspace2"));
nspFact22->SetFactory("Nullspace2", P22Fact);
RCP<CoarseMapFactory> coarseMapFact22 = Teuchos::rcp(new CoarseMapFactory());
coarseMapFact22->setStridingData(stridingInfo);
coarseMapFact22->setStridedBlockId(1);
//////////////////////////////// define factory manager for (2,2) block
RCP<FactoryManager> M22 = rcp(new FactoryManager());
M22->SetFactory("A", A22Fact);
M22->SetFactory("P", P22Fact);
M22->SetFactory("R", R22Fact);
M22->SetFactory("Aggregates", CoupledAggFact11);
M22->SetFactory("Nullspace", nspFact22);
M11->SetFactory("UnAmalgamationInfo", amalgFact22);
M22->SetFactory("Ptent", P22Fact);
M22->SetFactory("CoarseMap", coarseMapFact22);
M22->SetIgnoreUserData(true); // always use data from factories defined in factory manager
#endif
/////////////////////////////////////////// define blocked transfer ops
RCP<BlockedPFactory> PFact = rcp(new BlockedPFactory());
PFact->AddFactoryManager(M11);
PFact->AddFactoryManager(M22);
RCP<GenericRFactory> RFact = rcp(new GenericRFactory());
RFact->SetFactory("P", PFact);
RCP<Factory> AcFact = rcp(new BlockedRAPFactory());
AcFact->SetFactory("P", PFact);
AcFact->SetFactory("R", RFact);
*out << "Creating Simple Smoother" << std::endl;
//////////////////////////////////////////////////////////////////////
// Smoothers
RCP<SubBlockAFactory> A00Fact = Teuchos::rcp(new SubBlockAFactory());
A00Fact->SetFactory("A",MueLu::NoFactory::getRCP());
A00Fact->SetParameter("block row",Teuchos::ParameterEntry(0));
A00Fact->SetParameter("block col",Teuchos::ParameterEntry(0));
std::string ifpackTypePredictSmoother;
Teuchos::ParameterList ifpackListPredictSmoother;
ifpackListPredictSmoother.set("relaxation: sweeps", (LocalOrdinal) 1);
ifpackListPredictSmoother.set("relaxation: damping factor", (Scalar) 0.5);
ifpackTypePredictSmoother = "RELAXATION";
ifpackListPredictSmoother.set("relaxation: type", "Gauss-Seidel");
RCP<SmootherPrototype> smoProtoPredict = rcp( new TrilinosSmoother(ifpackTypePredictSmoother, ifpackListPredictSmoother, 0) );
smoProtoPredict->SetFactory("A", A00Fact);
RCP<SmootherFactory> SmooPredictFact = rcp( new SmootherFactory(smoProtoPredict) );
RCP<FactoryManager> MPredict = rcp(new FactoryManager());
MPredict->SetFactory("A", A00Fact); // SchurComplement operator for correction step (defined as "A")
MPredict->SetFactory("Smoother", SmooPredictFact); // solver/smoother for correction step
MPredict->SetFactory("PreSmoother", SmooPredictFact);
MPredict->SetFactory("PostSmoother", SmooPredictFact);
MPredict->SetIgnoreUserData(true); // always use data from factories defined in factory manager
////////////////////////////////////////////////
// SchurComp
// create SchurComp factory (SchurComplement smoother is provided by local FactoryManager)
Teuchos::RCP<SchurComplementFactory> SFact = Teuchos::rcp(new SchurComplementFactory());
SFact->SetParameter("omega", Teuchos::ParameterEntry(0.8));
SFact->SetParameter("lumping", Teuchos::ParameterEntry(true));
SFact->SetFactory("A", MueLu::NoFactory::getRCP()); // 2x2 blocked operator
// define SchurComplement solver
std::string ifpackTypeSchurSmoother;
ifpackTypeSchurSmoother = "RELAXATION";
Teuchos::ParameterList ifpackListSchurSmoother;
ifpackListSchurSmoother.set("relaxation: sweeps", (LocalOrdinal) 10);
ifpackListSchurSmoother.set("relaxation: damping factor", (Scalar) 0.8);
ifpackListSchurSmoother.set("relaxation: type", "Gauss-Seidel");
RCP<SmootherPrototype> smoProtoSC = rcp( new TrilinosSmoother(ifpackTypeSchurSmoother, ifpackListSchurSmoother, 0) );
smoProtoSC->SetFactory("A", SFact); // explicitely use SchurComplement matrix as input for smoother
RCP<SmootherFactory> SmooSCFact = rcp( new SmootherFactory(smoProtoSC) );
// setup local factory manager for SchurComplementFactory
Teuchos::RCP<FactoryManager> MSchur = Teuchos::rcp(new FactoryManager());
MSchur->SetFactory("A", SFact); // SchurCompFactory as generating factory for SchurComp equation
MSchur->SetFactory("Smoother", SmooSCFact);
MSchur->SetIgnoreUserData(true);
/////////////////////////////////////////////////////
// create smoother prototype
RCP<SimpleSmoother> smootherPrototype = rcp( new SimpleSmoother() );
smootherPrototype->SetParameter("Sweeps", Teuchos::ParameterEntry(3));
smootherPrototype->SetParameter("Damping factor", Teuchos::ParameterEntry(0.6));
smootherPrototype->AddFactoryManager(MPredict,0); // set temporary factory manager for prediction step
smootherPrototype->AddFactoryManager(MSchur,1); // set temporary factory manager for correction step
smootherPrototype->SetFactory("A", MueLu::NoFactory::getRCP());
/////////////////////////////////////////////////////
// create smoother factories
RCP<SmootherFactory> smootherFact = rcp( new SmootherFactory(smootherPrototype) ); // pre and postsmoothing with SIMPLE on the finest and intermedium levels
RCP<SmootherFactory> coarseSmootherFact = rcp( new SmootherFactory(smootherPrototype, Teuchos::null) ); // only presmoothing on finest level (we do not want to run two SIMPLE iterations on the coarsest level)
// main factory manager
FactoryManager M;
M.SetFactory("A", AcFact);
M.SetFactory("P", PFact);
M.SetFactory("R", RFact);
M.SetFactory("Smoother", smootherFact); // TODO fix me
M.SetFactory("CoarseSolver", coarseSmootherFact);
//////////////////////////////////// setup multigrid
H->Setup(M,0,maxLevels);
Finest->print(*out);
RCP<Level> coarseLevel = H->GetLevel(1);
coarseLevel->print(*out);
//RCP<Level> coarseLevel2 = H->GetLevel(2);
//coarseLevel2->print(*out);
RCP<MultiVector> xLsg = MultiVectorFactory::Build(xstridedfullmap,1);
// Use AMG directly as an iterative method
#if 0
{
xLsg->putScalar( (SC) 0.0);
// Epetra_Vector -> Xpetra::Vector
RCP<Vector> xRhs = Teuchos::rcp(new Xpetra::EpetraVector(epv));
// calculate initial (absolute) residual
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
xRhs->norm2(norms);
*out << "||x_0|| = " << norms[0] << std::endl;
// apply ten multigrid iterations
H->Iterate(*xRhs,*xLsg,100);
// calculate and print residual
RCP<MultiVector> xTmp = MultiVectorFactory::Build(xstridedfullmap,1);
bOp->apply(*xLsg,*xTmp,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
xRhs->update((SC)-1.0,*xTmp,(SC)1.0);
xRhs->norm2(norms);
*out << "||x|| = " << norms[0] << std::endl;
}
#endif
//
// Solve Ax = b using AMG as a preconditioner in AztecOO
//
{
RCP<Epetra_Vector> X = rcp(new Epetra_Vector(epv->Map()));
X->PutScalar(0.0);
Epetra_LinearProblem epetraProblem(epA.get(), X.get(), epv.get());
AztecOO aztecSolver(epetraProblem);
aztecSolver.SetAztecOption(AZ_solver, AZ_gmres);
MueLu::EpetraOperator aztecPrec(H);
aztecSolver.SetPrecOperator(&aztecPrec);
int maxIts = 50;
double tol = 1e-8;
aztecSolver.Iterate(maxIts, tol);
}
success = true;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
#else
std::cout << "Epetra (and/or EpetraExt) are not available. Skip test." << std::endl;
return EXIT_SUCCESS;
#endif // #if defined(HAVE_MUELU_SERIAL) && defined(HAVE_MUELU_EPETRA)
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int MyPID = comm->getRank();
int NumProc = comm->getSize();
const Teuchos::RCP<Epetra_Comm> epComm = Teuchos::rcp_const_cast<Epetra_Comm>(Xpetra::toEpetra(comm));
// =========================================================================
// Convenient definitions
// =========================================================================
//SC zero = Teuchos::ScalarTraits<SC>::zero(), one = Teuchos::ScalarTraits<SC>::one();
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100;
clp.setOption("nx", &nx, "mesh size in x direction");
clp.setOption("ny", &ny, "mesh size in y direction");
std::string xmlFileName = "xml/s3a.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'tutorial1a.xml'");
int mgridSweeps = 1; clp.setOption("mgridSweeps", &mgridSweeps, "number of multigrid sweeps within Multigrid solver.");
std::string printTimings = "no"; clp.setOption("timings", &printTimings, "print timings to screen [yes/no]");
double tol = 1e-12; clp.setOption("tol", &tol, "solver convergence tolerance");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
// =========================================================================
// Problem construction
// =========================================================================
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time"))), tm;
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
Teuchos::ParameterList GaleriList;
GaleriList.set("nx", nx);
GaleriList.set("ny", ny);
GaleriList.set("mx", epComm->NumProc());
GaleriList.set("my", 1);
GaleriList.set("lx", 1.0); // length of x-axis
GaleriList.set("ly", 1.0); // length of y-axis
GaleriList.set("diff", 1e-5);
GaleriList.set("conv", 1.0);
// create map
Teuchos::RCP<Epetra_Map> epMap = Teuchos::rcp(Galeri::CreateMap("Cartesian2D", *epComm, GaleriList));
// create coordinates
Teuchos::RCP<Epetra_MultiVector> epCoord = Teuchos::rcp(Galeri::CreateCartesianCoordinates("2D", epMap.get(), GaleriList));
// create matrix
Teuchos::RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(Galeri::CreateCrsMatrix("Recirc2D", epMap.get(), GaleriList));
// Epetra -> Xpetra
Teuchos::RCP<CrsMatrix> exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(epA));
Teuchos::RCP<CrsMatrixWrap> exAWrap = Teuchos::rcp(new CrsMatrixWrap(exA));
RCP<Matrix> A = Teuchos::rcp_dynamic_cast<Matrix>(exAWrap);
int numPDEs = 1;
A->SetFixedBlockSize(numPDEs);
// set rhs and solution vector
RCP<Epetra_Vector> B = Teuchos::rcp(new Epetra_Vector(*epMap));
RCP<Epetra_Vector> X = Teuchos::rcp(new Epetra_Vector(*epMap));
B->PutScalar(1.0);
X->PutScalar(0.0);
// Epetra -> Xpetra
RCP<Vector> xB = Teuchos::rcp(new Xpetra::EpetraVector(B));
RCP<Vector> xX = Teuchos::rcp(new Xpetra::EpetraVector(X));
xX->setSeed(100);
xX->randomize();
// build null space vector
RCP<const Map> map = A->getRowMap();
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, numPDEs);
for (int i=0; i<numPDEs; ++i) {
Teuchos::ArrayRCP<Scalar> nsValues = nullspace->getDataNonConst(i);
int numBlocks = nsValues.size() / numPDEs;
for (int j=0; j< numBlocks; ++j) {
nsValues[j*numPDEs + i] = 1.0;
}
}
comm->barrier();
tm = Teuchos::null;
fancyout << "========================================================\nGaleri complete.\n========================================================" << std::endl;
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
ParameterListInterpreter mueLuFactory(xmlFileName, *comm);
comm->barrier();
tm = Teuchos::null;
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
mueLuFactory.SetupHierarchy(*H);
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
//
// generate exact solution using a direct solver
//
RCP<Epetra_Vector> exactLsgVec = rcp(new Epetra_Vector(X->Map()));
{
fancyout << "========================================================\nCalculate exact solution." << std::endl;
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - direct solve")));
exactLsgVec->PutScalar(0.0);
exactLsgVec->Update(1.0,*X,1.0);
Epetra_LinearProblem epetraProblem(epA.get(), exactLsgVec.get(), B.get());
Amesos amesosFactory;
RCP<Amesos_BaseSolver> rcp_directSolver = Teuchos::rcp(amesosFactory.Create("Amesos_Klu", epetraProblem));
rcp_directSolver->SymbolicFactorization();
rcp_directSolver->NumericFactorization();
rcp_directSolver->Solve();
comm->barrier();
tm = Teuchos::null;
}
//
// Solve Ax = b using AMG as a preconditioner in AztecOO
//
RCP<Epetra_Vector> precLsgVec = rcp(new Epetra_Vector(X->Map()));
{
fancyout << "========================================================\nUse multigrid hierarchy as preconditioner within CG." << std::endl;
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - AMG as preconditioner")));
precLsgVec->PutScalar(0.0);
precLsgVec->Update(1.0,*X,1.0);
Epetra_LinearProblem epetraProblem(epA.get(), precLsgVec.get(), B.get());
AztecOO aztecSolver(epetraProblem);
aztecSolver.SetAztecOption(AZ_solver, AZ_gmres);
MueLu::EpetraOperator aztecPrec(H);
aztecSolver.SetPrecOperator(&aztecPrec);
int maxIts = 50;
aztecSolver.Iterate(maxIts, tol);
comm->barrier();
tm = Teuchos::null;
}
//////////////////
// use multigrid hierarchy as solver
RCP<Vector> mgridLsgVec = VectorFactory::Build(map);
mgridLsgVec->putScalar(0.0);
{
fancyout << "========================================================\nUse multigrid hierarchy as solver." << std::endl;
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Multigrid Solve")));
mgridLsgVec->update(1.0,*xX,1.0);
H->IsPreconditioner(false);
H->Iterate(*xB, mgridSweeps, *mgridLsgVec);
comm->barrier();
tm = Teuchos::null;
}
//////////////////
fancyout << "========================================================\nExport results.\n========================================================" << std::endl;
std::ofstream myfile;
std::stringstream ss; ss << "example" << MyPID << ".txt";
myfile.open (ss.str().c_str());
//////////////////
// loop over all procs
for (int iproc=0; iproc < NumProc; iproc++) {
if (MyPID==iproc) {
int NumVectors1 = 2;
int NumMyElements1 = epCoord->Map(). NumMyElements();
int MaxElementSize1 = epCoord->Map().MaxElementSize();
int * FirstPointInElementList1 = NULL;
if (MaxElementSize1!=1) FirstPointInElementList1 = epCoord->Map().FirstPointInElementList();
double ** A_Pointers = epCoord->Pointers();
if (MyPID==0) {
myfile.width(8);
myfile << "# MyPID"; myfile << " ";
myfile.width(12);
if (MaxElementSize1==1)
myfile << "GID ";
else
myfile << " GID/Point";
for (int j = 0; j < NumVectors1 ; j++)
{
myfile.width(20);
myfile << "Value ";
}
myfile << std::endl;
}
for (int i=0; i < NumMyElements1; i++) {
for (int ii=0; ii< epCoord->Map().ElementSize(i); ii++) {
int iii;
myfile.width(10);
myfile << MyPID; myfile << " ";
myfile.width(10);
if (MaxElementSize1==1) {
if(epCoord->Map().GlobalIndicesInt())
{
int * MyGlobalElements1 = epCoord->Map().MyGlobalElements();
myfile << MyGlobalElements1[i] << " ";
}
iii = i;
}
else {
if(epCoord->Map().GlobalIndicesInt())
{
int * MyGlobalElements1 = epCoord->Map().MyGlobalElements();
myfile << MyGlobalElements1[i]<< "/" << ii << " ";
}
iii = FirstPointInElementList1[i]+ii;
}
for (int j = 0; j < NumVectors1 ; j++)
{
myfile.width(20);
myfile << A_Pointers[j][iii];
}
myfile.precision(18); // set high precision for output
// add solution vector entry
myfile.width(25); myfile << (*exactLsgVec)[iii];
// add preconditioned solution vector entry
myfile.width(25); myfile << (*precLsgVec)[iii];
Teuchos::ArrayRCP<SC> mgridLsgVecData = mgridLsgVec->getDataNonConst(0);
myfile.width(25); myfile << mgridLsgVecData[iii];
myfile.precision(6); // set default precision
myfile << std::endl;
}
} // end loop over all lines on current proc
myfile << std::flush;
// syncronize procs
comm->barrier();
comm->barrier();
comm->barrier();
} // end myProc
}
////////////
myfile.close();
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings == "yes") {
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union);
}
return 0;
} //main
int main(int argc, char *argv[]) {
using Teuchos::RCP;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// Timing
Teuchos::Time myTime("global");
Teuchos::TimeMonitor M(myTime);
#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
#endif
// custom parameters
LO maxLevels = 10;
LO its=40;
std::string smooType="sgs";
int sweeps=1;
int maxCoarseSize=1; //FIXME clp doesn't like long long int
std::string aggOrdering = "natural";
int minPerAgg=2;
int maxNbrAlreadySelected=0;
// read in problem
Epetra_Map emap(10201,0,*Xpetra::toEpetra(comm));
Epetra_CrsMatrix * ptrA = 0;
Epetra_Vector * ptrf = 0;
std::cout << "Reading matrix market file" << std::endl;
EpetraExt::MatrixMarketFileToCrsMatrix("Condif2Mat.mat",emap,emap,emap,ptrA);
EpetraExt::MatrixMarketFileToVector("Condif2Rhs.mat",emap,ptrf);
RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
RCP<Epetra_Vector> epv = Teuchos::rcp(ptrf);
// Epetra_CrsMatrix -> Xpetra::Matrix
RCP<Xpetra::CrsMatrix<double, int, int> > exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(epA));
RCP<Xpetra::CrsMatrixWrap<double, int, int> > crsOp = Teuchos::rcp(new Xpetra::CrsMatrixWrap<double, int, int>(exA));
RCP<Xpetra::Matrix<double, int, int> > Op = Teuchos::rcp_dynamic_cast<Xpetra::Matrix<double, int, int> >(crsOp);
// Epetra_Vector -> Xpetra::Vector
RCP<Xpetra::Vector<double,int,int> > xRhs = Teuchos::rcp(new Xpetra::EpetraVector(epv));
// Epetra_Map -> Xpetra::Map
const RCP< const Xpetra::Map<int, int> > map = Xpetra::toXpetra(emap);
// build nullspace
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar( (SC) 1.0);
/*for (size_t i=0; i<nullSpace->getLocalLength(); i++) {
Teuchos::ArrayRCP< Scalar > data0 = nullSpace->getDataNonConst(0);
Teuchos::ArrayRCP< Scalar > data1 = nullSpace->getDataNonConst(1);
GlobalOrdinal gid = map->getGlobalElement(Teuchos::as<LocalOrdinal>(i));
if(gid % 2 == 0) {
data0[i] = 1.0; data1[i] = 0.0;
}
else {
data0[i] = 0.0; data1[i] = 1.0;
}
}*/
RCP<MueLu::Hierarchy<SC,LO,GO,NO,LMO> > H = rcp ( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
H->SetMaxCoarseSize((GO) maxCoarseSize);;
// build finest Level
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op);
Finest->Set("Nullspace",nullSpace);
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min DOFs per aggregate : " << minPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << maxNbrAlreadySelected << std::endl;
CoupledAggFact->SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D
CoupledAggFact->SetMaxNeighAlreadySelected(maxNbrAlreadySelected);
std::transform(aggOrdering.begin(), aggOrdering.end(), aggOrdering.begin(), ::tolower);
if (aggOrdering == "natural") {
*out << "aggregate ordering : NATURAL" << std::endl;
CoupledAggFact->SetOrdering(MueLu::AggOptions::NATURAL);
} else if (aggOrdering == "random") {
*out << "aggregate ordering : RANDOM" << std::endl;
CoupledAggFact->SetOrdering(MueLu::AggOptions::RANDOM);
} else if (aggOrdering == "graph") {
*out << "aggregate ordering : GRAPH" << std::endl;
CoupledAggFact->SetOrdering(MueLu::AggOptions::GRAPH);
} else {
std::string msg = "main: bad aggregation option """ + aggOrdering + """.";
throw(MueLu::Exceptions::RuntimeError(msg));
}
CoupledAggFact->SetPhase3AggCreation(0.5);
*out << "=============================================================================" << std::endl;
// build transfer operators
RCP<TentativePFactory> TentPFact = rcp(new TentativePFactory(CoupledAggFact));
*out << " afer TentativePFactory " << std::endl;
//RCP<TentativePFactory> Pfact = rcp(new TentativePFactory(CoupledAggFact));
//RCP<Factory> Rfact = rcp( new TransPFactory(Pfact));
//RCP<SaPFactory> Pfact = rcp( new SaPFactory(TentPFact) );
//RCP<Factory> Rfact = rcp( new TransPFactory(Pfact));
RCP<ThresholdAFilterFactory> Afiltered = rcp(new ThresholdAFilterFactory("A",NULL,0.0005));
RCP<PgPFactory> Pfact = rcp( new PgPFactory(TentPFact,Afiltered) );
RCP<Factory> Rfact = rcp( new GenericRFactory(Pfact));
RCP<RAPFactory> Acfact = rcp( new RAPFactory(Pfact, Rfact) );
Acfact->setVerbLevel(Teuchos::VERB_HIGH);
*out << " after ACFactory " << std::endl;
// build level smoothers
RCP<SmootherPrototype> smooProto;
std::string ifpackType;
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) sweeps);
ifpackList.set("relaxation: damping factor", (SC) 0.9); // 0.7
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Gauss-Seidel");
smooProto = Teuchos::rcp( new TrilinosSmoother(Xpetra::UseEpetra, ifpackType, ifpackList) );
RCP<SmootherFactory> SmooFact;
if (maxLevels > 1)
SmooFact = rcp( new SmootherFactory(smooProto) );
Teuchos::ParameterList status;
#if 0 // both variants are equivalent
// fill FactoryManager object by hand
FactoryManager Manager;
Manager.SetFactory("A", Acfact);
Manager.SetFactory("P", Pfact);
Manager.SetFactory("R", Rfact);
Manager.SetFactory("Smoother", SmooFact);
Manager.SetFactory("CoarseSolver", Teuchos::null);
status = H->Setup(Manager);
#else
// use FullPopulate (short, but outdated?)
status = H->FullPopulate(*Pfact,*Rfact,*Acfact,*SmooFact,0,maxLevels);
#endif
H->SetCoarsestSolver(*SmooFact,MueLu::PRE);
*out << "======================\n Multigrid statistics \n======================" << std::endl;
status.print(*out,Teuchos::ParameterList::PrintOptions().indent(2));
/**********************************************************************************/
/* SOLVE PROBLEM */
/**********************************************************************************/
RCP<MultiVector> x = MultiVectorFactory::Build(map,1);
RCP<Xpetra::MultiVector<double,int,int> > rhs = Teuchos::rcp_dynamic_cast<Xpetra::MultiVector<double,int,int> >(xRhs);//(new Xpetra::EpetraVector(epv));
// Use AMG directly as an iterative method
{
x->putScalar( (SC) 0.0);
H->Iterate(*rhs,its,*x);
//x->describe(*out,Teuchos::VERB_EXTREME);
}
RCP<Level> coarseLevel = H->GetLevel(1);
coarseLevel->print(*out);
RCP<Level> coarseLevel2 = H->GetLevel(1);
coarseLevel2->print(*out);
//M.summarize();
return EXIT_SUCCESS;
}
TEUCHOS_UNIT_TEST(MultiVectorTransferFactory, ThreeLevels)
{
out << "version: " << MueLu::Version() << std::endl;
out << "Tests usage on a three-level hierarchy." << std::endl;
GO nx = 199;
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::Build1DPoisson(nx);
// Set up three level hierarchy.
RCP<Hierarchy> H = rcp( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
RCP<Level> fineLevel = H->GetLevel();
fineLevel->setDefaultVerbLevel(Teuchos::VERB_HIGH);
fineLevel->Set("A",A); // set fine level matrix
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(A->getRowMap(),1);
nullSpace->putScalar( (SC) 1.0);
fineLevel->Set("Nullspace",nullSpace); // set null space information for finest level
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetMinNodesPerAggregate(3);
CoupledAggFact->SetMaxNeighAlreadySelected(0);
CoupledAggFact->SetOrdering(MueLu::AggOptions::NATURAL);
CoupledAggFact->SetPhase3AggCreation(0.5);
RCP<TentativePFactory> PFact = rcp(new TentativePFactory()); //just using plain aggregation
RCP<Factory> RFact = rcp(new TransPFactory());
RCP<RAPFactory> AcFact = rcp(new RAPFactory());
H->SetMaxCoarseSize(1);
Teuchos::ParameterList smootherParamList;
smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
smootherParamList.set("relaxation: sweeps", (LO) 1);
smootherParamList.set("relaxation: damping factor", (SC) 1.0);
RCP<SmootherPrototype> smooProto = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );
RCP<SmootherFactory> SmooFact = rcp( new SmootherFactory(smooProto) );
AcFact->setVerbLevel(Teuchos::VERB_HIGH);
FactoryManager M;
M.SetFactory("Aggregates", CoupledAggFact);
M.SetFactory("P", PFact);
M.SetFactory("Ptent", PFact); // for nullspace
M.SetFactory("R", RFact);
M.SetFactory("A", AcFact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", SmooFact); // This line avoid dependency to Amesos/Amesos2 for this test.
//set up the transfer factory
RCP<MultiVector> fineOnes = MultiVectorFactory::Build(A->getRowMap(),1);
fineOnes->putScalar(1.0);
fineLevel->Set("onesVector",fineOnes);
RCP<MueLu::MultiVectorTransferFactory<SC, LO, GO, NO, LMO> > mvtf = rcp(new MueLu::MultiVectorTransferFactory<SC, LO, GO, NO, LMO>("onesVector"));
mvtf->SetFactory("R",RFact);
M.SetFactory("onesVector",mvtf);
AcFact->AddTransferFactory(mvtf);
int maxLevels = 3;
H->Setup(M, 0, maxLevels);
/*
//FIXME we probably need to do some requests....
coarseLevel.Request("onesVector",mvtf.get());
coarseLevel.Request("R",RFact.get());
coarseLevel.Request("P",TentativePFact.get());
*/
/*
RCP<MultiVector> coarseOnes = coarseLevel.Get<RCP<MultiVector> >("onesVector",mvtf.get());
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> vn(1);
coarseOnes->norm2(vn);
TEST_FLOATING_EQUALITY(vn[0]*vn[0],((SC)fineOnes->getGlobalLength()),1e-12);
*/
} // ThreeLevels
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
typedef Tpetra::Vector<SC,LO,GO,NO> TVEC;
typedef Tpetra::MultiVector<SC,LO,GO,NO> TMV;
typedef Tpetra::CrsMatrix<SC,LO,GO,NO,LMO> TCRS;
typedef Xpetra::CrsMatrix<SC,LO,GO,NO,LMO> XCRS;
typedef Xpetra::TpetraCrsMatrix<SC,LO,GO,NO,LMO> XTCRS;
typedef Xpetra::Matrix<SC,LO,GO,NO,LMO> XMAT;
typedef Xpetra::CrsMatrixWrap<SC,LO,GO,NO,LMO> XWRAP;
typedef Belos::OperatorT<TMV> TOP;
typedef Belos::OperatorTraits<SC,TMV,TOP> TOPT;
typedef Belos::MultiVecTraits<SC,TMV> TMVT;
typedef Belos::LinearProblem<SC,TMV,TOP> TProblem;
typedef Belos::SolverManager<SC,TMV,TOP> TBelosSolver;
typedef Belos::BlockGmresSolMgr<SC,TMV,TOP> TBelosGMRES;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
Teuchos::CommandLineProcessor clp(false);
GO nx,ny,nz;
nx=100; ny=100; nz=100;
double stretchx, stretchy, stretchz, h, delta;
stretchx=1.0; stretchy=1.0; stretchz=1.0;
h=0.01; delta=2.0;
int PMLXL, PMLXR, PMLYL, PMLYR, PMLZL, PMLZR;
PMLXL=10; PMLXR=10; PMLYL=10; PMLYR=10; PMLZL=10; PMLZR=10;
double omega, shift;
omega=20.0*M_PI;
shift=0.5;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Helmholtz1D", 0, stretchx, stretchy, stretchz,
h, delta, PMLXL, PMLXR, PMLYL, PMLYR, PMLZL, PMLZR, omega, shift);
Xpetra::Parameters xpetraParameters(clp);
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
RCP<TimeMonitor> tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
RCP<MultiVector> coordinates;
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx", nx));
galeriList.set("ny", pl.get("ny", ny));
galeriList.set("nz", pl.get("nz", nz));
RCP<const Map> map;
if (matrixParameters.GetMatrixType() == "Helmholtz1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Helmholtz2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Helmholtz3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
RCP<const Tpetra::Map<LO, GO, NO> > tmap = Xpetra::toTpetra(map);
Teuchos::ParameterList matrixParams = matrixParameters.GetParameterList();
// Build problem
RCP<Galeri::Xpetra::Problem_Helmholtz<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem_Helmholtz<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParams);
RCP<Matrix> A = Pr->BuildMatrix();
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map,1);
nullspace->putScalar( (SC) 1.0);
comm->barrier();
tm = Teuchos::null;
// Construct a multigrid preconditioner
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
// Multigrid Hierarchy
RCP<Hierarchy> H = rcp(new Hierarchy(A));
H->GetLevel(0)->Set("Nullspace",nullspace);
FactoryManager Manager;
H->Setup(Manager, 0, 5);
//H->Write(-1,-1);
tm = Teuchos::null;
// Solve Ax = b
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<TVEC> X = Tpetra::createVector<SC,LO,GO,NO>(tmap);
RCP<TVEC> B = Tpetra::createVector<SC,LO,GO,NO>(tmap);
X->putScalar((SC) 0.0);
B->putScalar((SC) 0.0);
if(comm->getRank()==0) {
B->replaceGlobalValue(0, (SC) 1.0);
}
tm = Teuchos::null;
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Belos Solve")));
// Define Operator and Preconditioner
RCP<TOP> belosOp = rcp(new Belos::XpetraOp<SC,LO,GO,NO,LMO> (A) ); // Turns a Xpetra::Matrix object into a Belos operator
RCP<TOP> belosPrec = rcp(new Belos::MueLuOp<SC,LO,GO,NO,LMO> (H) ); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP<TProblem> belosProblem = rcp(new TProblem(belosOp,X,B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
if(comm->getRank()==0)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 100;
double tol = 1e-6;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Flexible Gmres", false); // set flexible GMRES on/off
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency",1);
belosList.set("Output Style",Belos::Brief);
// Create solver manager
RCP<TBelosSolver> solver = rcp( new TBelosGMRES(belosProblem, rcp(&belosList, false)) );
// Perform solve
Belos::ReturnType ret=Belos::Unconverged;
try {
ret = solver->solve();
if (comm->getRank() == 0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
}
catch(...) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged) {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
} else {
if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
}
// Get the number of iterations for this solve.
if(comm->getRank()==0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
TimeMonitor::summarize();
} //main
void MueLuPreconditionerFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node>::
initializePrec(const RCP<const LinearOpSourceBase<Scalar> >& fwdOpSrc, PreconditionerBase<Scalar>* prec, const ESupportSolveUse supportSolveUse) const {
using Teuchos::rcp_dynamic_cast;
// we are using typedefs here, since we are using objects from different packages (Xpetra, Thyra,...)
typedef Xpetra::Map<LocalOrdinal,GlobalOrdinal,Node> XpMap;
typedef Xpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> XpOp;
typedef Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpThyUtils;
typedef Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpCrsMat;
typedef Xpetra::BlockedCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpBlockedCrsMat;
typedef Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpMat;
typedef Xpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpMultVec;
typedef Xpetra::MultiVector<double,LocalOrdinal,GlobalOrdinal,Node> XpMultVecDouble;
typedef Thyra::LinearOpBase<Scalar> ThyLinOpBase;
#ifdef HAVE_MUELU_TPETRA
typedef MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> MueTpOp;
typedef Tpetra::Operator<Scalar,LocalOrdinal,GlobalOrdinal,Node> TpOp;
typedef Thyra::TpetraLinearOp<Scalar,LocalOrdinal,GlobalOrdinal,Node> ThyTpLinOp;
#endif
// Check precondition
TEUCHOS_ASSERT(Teuchos::nonnull(fwdOpSrc));
TEUCHOS_ASSERT(this->isCompatible(*fwdOpSrc));
TEUCHOS_ASSERT(prec);
// Create a copy, as we may remove some things from the list
ParameterList paramList = *paramList_;
// Retrieve wrapped concrete Xpetra matrix from FwdOp
const RCP<const ThyLinOpBase> fwdOp = fwdOpSrc->getOp();
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(fwdOp));
// Check whether it is Epetra/Tpetra
bool bIsEpetra = XpThyUtils::isEpetra(fwdOp);
bool bIsTpetra = XpThyUtils::isTpetra(fwdOp);
bool bIsBlocked = XpThyUtils::isBlockedOperator(fwdOp);
TEUCHOS_TEST_FOR_EXCEPT((bIsEpetra == true && bIsTpetra == true));
TEUCHOS_TEST_FOR_EXCEPT((bIsEpetra == bIsTpetra) && bIsBlocked == false);
TEUCHOS_TEST_FOR_EXCEPT((bIsEpetra != bIsTpetra) && bIsBlocked == true);
RCP<XpMat> A = Teuchos::null;
if(bIsBlocked) {
Teuchos::RCP<const Thyra::BlockedLinearOpBase<Scalar> > ThyBlockedOp =
Teuchos::rcp_dynamic_cast<const Thyra::BlockedLinearOpBase<Scalar> >(fwdOp);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(ThyBlockedOp));
TEUCHOS_TEST_FOR_EXCEPT(ThyBlockedOp->blockExists(0,0)==false);
Teuchos::RCP<const LinearOpBase<Scalar> > b00 = ThyBlockedOp->getBlock(0,0);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(b00));
RCP<const XpCrsMat > xpetraFwdCrsMat00 = XpThyUtils::toXpetra(b00);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMat00));
// MueLu needs a non-const object as input
RCP<XpCrsMat> xpetraFwdCrsMatNonConst00 = Teuchos::rcp_const_cast<XpCrsMat>(xpetraFwdCrsMat00);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMatNonConst00));
// wrap the forward operator as an Xpetra::Matrix that MueLu can work with
RCP<XpMat> A00 = rcp(new Xpetra::CrsMatrixWrap<Scalar,LocalOrdinal,GlobalOrdinal,Node>(xpetraFwdCrsMatNonConst00));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(A00));
RCP<const XpMap> rowmap00 = A00->getRowMap();
RCP< const Teuchos::Comm< int > > comm = rowmap00->getComm();
// create a Xpetra::BlockedCrsMatrix which derives from Xpetra::Matrix that MueLu can work with
RCP<XpBlockedCrsMat> bMat = Teuchos::rcp(new XpBlockedCrsMat(ThyBlockedOp, comm));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(bMat));
// save blocked matrix
A = bMat;
} else {
RCP<const XpCrsMat > xpetraFwdCrsMat = XpThyUtils::toXpetra(fwdOp);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMat));
// MueLu needs a non-const object as input
RCP<XpCrsMat> xpetraFwdCrsMatNonConst = Teuchos::rcp_const_cast<XpCrsMat>(xpetraFwdCrsMat);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMatNonConst));
// wrap the forward operator as an Xpetra::Matrix that MueLu can work with
A = rcp(new Xpetra::CrsMatrixWrap<Scalar,LocalOrdinal,GlobalOrdinal,Node>(xpetraFwdCrsMatNonConst));
}
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(A));
// Retrieve concrete preconditioner object
const Teuchos::Ptr<DefaultPreconditioner<Scalar> > defaultPrec = Teuchos::ptr(dynamic_cast<DefaultPreconditioner<Scalar> *>(prec));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(defaultPrec));
// extract preconditioner operator
RCP<ThyLinOpBase> thyra_precOp = Teuchos::null;
thyra_precOp = rcp_dynamic_cast<Thyra::LinearOpBase<Scalar> >(defaultPrec->getNonconstUnspecifiedPrecOp(), true);
// Variable for multigrid hierarchy: either build a new one or reuse the existing hierarchy
RCP<MueLu::Hierarchy<Scalar,LocalOrdinal,GlobalOrdinal,Node> > H = Teuchos::null;
// make a decision whether to (re)build the multigrid preconditioner or reuse the old one
// rebuild preconditioner if startingOver == true
// reuse preconditioner if startingOver == false
const bool startingOver = (thyra_precOp.is_null() || !paramList.isParameter("reuse: type") || paramList.get<std::string>("reuse: type") == "none");
if (startingOver == true) {
// extract coordinates from parameter list
Teuchos::RCP<XpMultVecDouble> coordinates = Teuchos::null;
coordinates = MueLu::Utilities<Scalar,LocalOrdinal,GlobalOrdinal,Node>::ExtractCoordinatesFromParameterList(paramList);
// TODO check for Xpetra or Thyra vectors?
RCP<XpMultVec> nullspace = Teuchos::null;
#ifdef HAVE_MUELU_TPETRA
if (bIsTpetra) {
typedef Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> tMV;
RCP<tMV> tpetra_nullspace = Teuchos::null;
if (paramList.isType<Teuchos::RCP<tMV> >("Nullspace")) {
tpetra_nullspace = paramList.get<RCP<tMV> >("Nullspace");
paramList.remove("Nullspace");
nullspace = MueLu::TpetraMultiVector_To_XpetraMultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node>(tpetra_nullspace);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(nullspace));
}
}
#endif
// build a new MueLu hierarchy
H = MueLu::CreateXpetraPreconditioner(A, paramList, coordinates, nullspace);
} else {
// reuse old MueLu hierarchy stored in MueLu Tpetra/Epetra operator and put in new matrix
// get old MueLu hierarchy
#if defined(HAVE_MUELU_TPETRA)
if (bIsTpetra) {
RCP<ThyTpLinOp> tpetr_precOp = rcp_dynamic_cast<ThyTpLinOp>(thyra_precOp);
RCP<MueTpOp> muelu_precOp = rcp_dynamic_cast<MueTpOp>(tpetr_precOp->getTpetraOperator(),true);
H = muelu_precOp->GetHierarchy();
}
#endif
// TODO add the blocked matrix case here...
TEUCHOS_TEST_FOR_EXCEPTION(!H->GetNumLevels(), MueLu::Exceptions::RuntimeError,
"Thyra::MueLuPreconditionerFactory: Hierarchy has no levels in it");
TEUCHOS_TEST_FOR_EXCEPTION(!H->GetLevel(0)->IsAvailable("A"), MueLu::Exceptions::RuntimeError,
"Thyra::MueLuPreconditionerFactory: Hierarchy has no fine level operator");
RCP<MueLu::Level> level0 = H->GetLevel(0);
RCP<XpOp> O0 = level0->Get<RCP<XpOp> >("A");
RCP<XpMat> A0 = rcp_dynamic_cast<XpMat>(O0);
if (!A0.is_null()) {
// If a user provided a "number of equations" argument in a parameter list
// during the initial setup, we must honor that settings and reuse it for
// all consequent setups.
A->SetFixedBlockSize(A0->GetFixedBlockSize());
}
// set new matrix
level0->Set("A", A);
H->SetupRe();
}
// wrap hierarchy H in thyraPrecOp
RCP<ThyLinOpBase > thyraPrecOp = Teuchos::null;
#if defined(HAVE_MUELU_TPETRA)
if (bIsTpetra) {
RCP<MueTpOp> muelu_tpetraOp = rcp(new MueTpOp(H));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(muelu_tpetraOp));
RCP<TpOp> tpOp = Teuchos::rcp_dynamic_cast<TpOp>(muelu_tpetraOp);
thyraPrecOp = Thyra::createLinearOp<Scalar, LocalOrdinal, GlobalOrdinal, Node>(tpOp);
}
#endif
if(bIsBlocked) {
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::nonnull(thyraPrecOp));
typedef MueLu::XpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> MueXpOp;
//typedef Thyra::XpetraLinearOp<Scalar,LocalOrdinal,GlobalOrdinal,Node> ThyXpLinOp; // unused
const RCP<MueXpOp> muelu_xpetraOp = rcp(new MueXpOp(H));
RCP<const VectorSpaceBase<Scalar> > thyraRangeSpace = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyra(muelu_xpetraOp->getRangeMap());
RCP<const VectorSpaceBase<Scalar> > thyraDomainSpace = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyra(muelu_xpetraOp->getDomainMap());
RCP <Xpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> > xpOp = Teuchos::rcp_dynamic_cast<Xpetra::Operator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >(muelu_xpetraOp);
thyraPrecOp = Thyra::xpetraLinearOp(thyraRangeSpace, thyraDomainSpace,xpOp);
}
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(thyraPrecOp));
defaultPrec->initializeUnspecified(thyraPrecOp);
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
bool success = false;
bool verbose = true;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int numProc = comm->getSize();
int myRank = comm->getRank();
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& out = *fancy;
out.setOutputToRootOnly(0);
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
Xpetra::Parameters xpetraParameters(clp);
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
const int numLists = 1;
std::vector<std::string> dirList;
dirList.push_back("Convergence/Laplace2D/");
bool failed = false;
for (int k = 0; k < numLists; k++) {
const std::string& dirName = dirList[k];
std::string problemFile = dirName + "problem.xml";
ParameterList galeriParameters;
Teuchos::updateParametersFromXmlFileAndBroadcast(problemFile, Teuchos::Ptr<Teuchos::ParameterList>(&galeriParameters), *comm);
if (!galeriParameters.isParameter("mz"))
galeriParameters.set<int>("mz", -1);
// =========================================================================
// Problem construction (copy-paste from Driver.cpp)
// =========================================================================
RCP<Matrix> A;
RCP<Map> map;
RCP<MultiVector> nullspace, coordinates;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriParameters.
std::string matrixType = galeriParameters.get<std::string>("matrixType");
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian1D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriParameters);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian2D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriParameters);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian3D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriParameters);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
#if 0
out << "========================================================\n" << xpetraParameters << galeriParameters;
out << "Processor subdomains in x direction: " << galeriParameters.get<GO>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriParameters.get<GO>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriParameters.get<GO>("mz") << std::endl
<< "========================================================" << std::endl;
#endif
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixType, map, galeriParameters);
A = Pr->BuildMatrix();
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((matrixType == "Elasticity2D") ? 2 : 3);
} else {
nullspace = MultiVectorFactory::Build(map, 1);
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(0);
for (int i = 0; i < nsData.size(); i++)
nsData[i] = one;
}
// =========================================================================
// Run different configurations
// =========================================================================
Teuchos::ArrayRCP<std::string> fileList = MueLuTests::TestHelpers::GetFileList(dirList[k],
(numProc == 1 ? std::string(".xml") : std::string("_np" + Teuchos::toString(numProc) + ".xml")));
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
for (int i = 0; i < fileList.size(); i++) {
if (fileList[i] == "problem.xml")
continue;
// Set seed
Utilities::SetRandomSeed(*comm);
// Reset (potentially) cached value of the estimate
A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());
std::string xmlFile = dirName + fileList[i];
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFile, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
std::string solveType = paramList.get<std::string> ("solver", "standalone");
double goldRate = paramList.get<double> ("convergence rate");
ParameterList& mueluList = paramList.sublist ("MueLu");
TEUCHOS_TEST_FOR_EXCEPTION(solveType != "standalone" && solveType != "cg" && solveType != "gmres", MueLu::Exceptions::RuntimeError,
"Unknown solver type \"" << solveType << "\"");
bool isPrec = !(solveType == "standalone");
if (!mueluList.isParameter("verbosity"))
mueluList.set("verbosity", "none");
#ifndef HAVE_MUELU_BELOS
if (isPrec)
out << xmlFile << ": skipped (Belos is not enabled)" << std::endl;
#endif
// =========================================================================
// Preconditioner construction
// =========================================================================
RCP<Hierarchy> H;
try {
ParameterListInterpreter mueluFactory(mueluList);
H = mueluFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("Coordinates", coordinates);
mueluFactory.SetupHierarchy(*H);
} catch (Teuchos::ExceptionBase& e) {
std::string msg = e.what();
msg = msg.substr(msg.find_last_of('\n')+1);
out << "Caught exception: " << msg << std::endl;
if (msg == "Zoltan interface is not available" ||
msg == "Zoltan2 interface is not available") {
if (myRank == 0)
out << xmlFile << ": skipped (missing library)" << std::endl;
continue;
}
}
// Set X, B
{
// TODO: do multiple vectors simultaneously to average
// we set seed for reproducibility
Utilities::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
const int maxIts = 100;
const double tol = 1e-12;
H->IsPreconditioner(isPrec);
if (isPrec == false) {
MueLu::ReturnType ret = H->Iterate(*B, *X, std::pair<LO,SC>(maxIts, tol));
double rate = H->GetRate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} else {
#ifdef HAVE_MUELU_BELOS
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
// Define Operator and Preconditioner
RCP<OP> belosOp = rcp(new Belos::XpetraOp<SC, LO, GO, NO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
RCP<OP> belosPrec = rcp(new Belos::MueLuOp <SC, LO, GO, NO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
out << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
#if 1
belosList.set("Verbosity", Belos::Errors + Belos::Warnings);
#else
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
#endif
// Belos custom test to store residuals
// Create an iterative solver manager
RCP<Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg")
solver = rcp(new Belos::PseudoBlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
else if (solveType == "gmres")
solver = rcp(new Belos::BlockGmresSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
RCP<Belos::MyStatusTest<SC, MV, OP> > status = rcp(new Belos::MyStatusTest<SC, MV, OP>(tol));
solver->setDebugStatusTest(status);
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
double rate = status->rate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} catch(...) {
out << xmlFile << ": failed (exception)" << std::endl;
failed = true;
}
#endif //ifdef HAVE_MUELU_BELOS
}
}
}
success = !failed;
out << std::endl << "End Result: TEST " << (failed ? "FAILED" : "PASSED") << std::endl;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::CrsMatrix <Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inA,
Teuchos::ParameterList& paramListIn,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> >& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
bool hasParamList = paramListIn.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = paramListIn;
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(inA);
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
int main(int argc, char *argv[]) {
#if defined(HAVE_MUELU_EPETRA) && defined(HAVE_MUELU_EPETRAEXT)
typedef double Scalar;
typedef int LocalOrdinal;
typedef int GlobalOrdinal;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Xpetra::EpetraNode Node;
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP;
using Teuchos::rcp;
using namespace MueLuTests;
using namespace Teuchos;
oblackholestream blackhole;
GlobalMPISession mpiSession(&argc,&argv,&blackhole);
bool success = false;
bool verbose = true;
try {
// default parameters
std::string xmlFile = "myXML.xml";
// Note: use --help to list available options.
CommandLineProcessor clp(false);
clp.setOption("xml", &xmlFile, "xml file with solver parameters for a 2x2 blocked NS example");
switch (clp.parse(argc,argv)) {
case CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case CommandLineProcessor::PARSE_ERROR:
case CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
RCP<const Comm<int> > comm = DefaultComm<int>::getComm();
RCP<FancyOStream> out = fancyOStream(rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// Timing
Time myTime("global");
TimeMonitor MM(myTime);
GO maxCoarseSize=1; //FIXME clp doesn't like long long int
int globalNumDofs = 1500; // used for the maps
int nDofsPerNode = 3; // used for generating the fine level null-space
// build strided maps
// striding information: 2 velocity dofs and 1 pressure dof = 3 dofs per node
std::vector<size_t> stridingInfo;
stridingInfo.push_back(2);
stridingInfo.push_back(1);
/////////////////////////////////////// build strided maps
// build strided maps:
// xstridedfullmap: full map (velocity and pressure dof gids), continous
// xstridedvelmap: only velocity dof gid maps (i.e. 0,1,3,4,6,7...)
// xstridedpremap: only pressure dof gid maps (i.e. 2,5,8,...)
Xpetra::UnderlyingLib lib = Xpetra::UseEpetra;
RCP<const StridedMap> xstridedfullmap = StridedMapFactory::Build(lib,globalNumDofs,0,stridingInfo,comm,-1);
RCP<const StridedMap> xstridedvelmap = StridedMapFactory::Build(xstridedfullmap,0);
RCP<const StridedMap> xstridedpremap = StridedMapFactory::Build(xstridedfullmap,1);
/////////////////////////////////////// transform Xpetra::Map objects to Epetra
// this is needed for AztecOO
const RCP<const Epetra_Map> fullmap = rcpFromRef(Xpetra::toEpetra(*xstridedfullmap));
RCP<const Epetra_Map> velmap = rcpFromRef(Xpetra::toEpetra(*xstridedvelmap));
RCP<const Epetra_Map> premap = rcpFromRef(Xpetra::toEpetra(*xstridedpremap));
/////////////////////////////////////// import problem matrix and RHS from files (-> Epetra)
// read in problem
Epetra_CrsMatrix * ptrA = 0;
Epetra_Vector * ptrf = 0;
Epetra_MultiVector* ptrNS = 0;
*out << "Reading matrix market file" << std::endl;
EpetraExt::MatrixMarketFileToCrsMatrix("A_re1000_5932.txt",*fullmap,*fullmap,*fullmap,ptrA);
EpetraExt::MatrixMarketFileToVector("b_re1000_5932.txt",*fullmap,ptrf);
RCP<Epetra_CrsMatrix> epA = rcp(ptrA);
RCP<Epetra_Vector> epv = rcp(ptrf);
RCP<Epetra_MultiVector> epNS = rcp(ptrNS);
/////////////////////////////////////// split system into 2x2 block system
*out << "Split matrix into 2x2 block matrix" << std::endl;
// split fullA into A11,..., A22
RCP<Epetra_CrsMatrix> A11;
RCP<Epetra_CrsMatrix> A12;
RCP<Epetra_CrsMatrix> A21;
RCP<Epetra_CrsMatrix> A22;
if(SplitMatrix2x2(epA,*velmap,*premap,A11,A12,A21,A22)==false)
*out << "Problem with splitting matrix"<< std::endl;
/////////////////////////////////////// transform Epetra objects to Xpetra (needed for MueLu)
// build Xpetra objects from Epetra_CrsMatrix objects
RCP<Xpetra::CrsMatrix<Scalar,LO,GO,Node> > xA11 = rcp(new Xpetra::EpetraCrsMatrixT<GO,Node>(A11));
RCP<Xpetra::CrsMatrix<Scalar,LO,GO,Node> > xA12 = rcp(new Xpetra::EpetraCrsMatrixT<GO,Node>(A12));
RCP<Xpetra::CrsMatrix<Scalar,LO,GO,Node> > xA21 = rcp(new Xpetra::EpetraCrsMatrixT<GO,Node>(A21));
RCP<Xpetra::CrsMatrix<Scalar,LO,GO,Node> > xA22 = rcp(new Xpetra::EpetraCrsMatrixT<GO,Node>(A22));
/////////////////////////////////////// generate MapExtractor object
std::vector<RCP<const Xpetra::Map<LO,GO,Node> > > xmaps;
xmaps.push_back(xstridedvelmap);
xmaps.push_back(xstridedpremap);
RCP<const Xpetra::MapExtractor<Scalar,LO,GO,Node> > map_extractor = Xpetra::MapExtractorFactory<Scalar,LO,GO,Node>::Build(xstridedfullmap,xmaps);
/////////////////////////////////////// build blocked transfer operator
// using the map extractor
RCP<Xpetra::BlockedCrsMatrix<Scalar,LO,GO,Node> > bOp = rcp(new Xpetra::BlockedCrsMatrix<Scalar,LO,GO,Node>(map_extractor,map_extractor,10));
bOp->setMatrix(0,0,xA11);
bOp->setMatrix(0,1,xA12);
bOp->setMatrix(1,0,xA21);
bOp->setMatrix(1,1,xA22);
bOp->fillComplete();
//////////////////////////////////////// prepare setup
ParameterListInterpreter mueLuFactory(xmlFile, *comm);
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
H->setDefaultVerbLevel(VERB_HIGH);
H->SetMaxCoarseSize(maxCoarseSize);
RCP<MueLu::Level> Finest = H->GetLevel(0);
Finest->setDefaultVerbLevel(VERB_HIGH);
Finest->Set("A", rcp_dynamic_cast<Matrix>(bOp));
////////////////////////////////////////// prepare null space for A11
RCP<MultiVector> nullspace11 = MultiVectorFactory::Build(xstridedvelmap, 2); // this is a 2D standard null space
for (int i=0; i<nDofsPerNode-1; ++i) {
ArrayRCP<Scalar> nsValues = nullspace11->getDataNonConst(i);
int numBlocks = nsValues.size() / (nDofsPerNode - 1);
for (int j=0; j< numBlocks; ++j) {
nsValues[j*(nDofsPerNode - 1) + i] = 1.0;
}
}
Finest->Set("Nullspace1",nullspace11);
////////////////////////////////////////// prepare null space for A22
RCP<MultiVector> nullspace22 = MultiVectorFactory::Build(xstridedpremap, 1); // this is a 2D standard null space
ArrayRCP<Scalar> nsValues22 = nullspace22->getDataNonConst(0);
for (int j=0; j< nsValues22.size(); ++j) {
nsValues22[j] = 1.0;
}
Finest->Set("Nullspace2",nullspace22);
/////////////////////////////////// BEGIN setup
mueLuFactory.SetupHierarchy(*H);
///////////////////////////////////// END setup
*out << std::endl;
RCP<MultiVector> xLsg = MultiVectorFactory::Build(xstridedfullmap,1);
// Use AMG directly as an iterative method
{
xLsg->putScalar( (SC) 0.0);
// Epetra_Vector -> Xpetra::Vector
RCP<Vector> xRhs = rcp(new Xpetra::EpetraVectorT<int,Node>(epv));
// calculate initial (absolute) residual
Array<ScalarTraits<SC>::magnitudeType> norms(1);
xRhs->norm2(norms);
*out << "||x_0|| = " << norms[0] << std::endl;
// apply ten multigrid iterations
H->Iterate(*xRhs,*xLsg,100);
// calculate and print residual
RCP<MultiVector> xTmp = MultiVectorFactory::Build(xstridedfullmap,1);
bOp->apply(*xLsg,*xTmp,NO_TRANS,(SC)1.0,(SC)0.0);
xRhs->update((SC)-1.0,*xTmp,(SC)1.0);
xRhs->norm2(norms);
*out << "||r|| = " << norms[0] << std::endl;
}
// TODO: don't forget to add Aztec as prerequisite in CMakeLists.txt!
//
// Solve Ax = b using AMG as a preconditioner in AztecOO
//
{
RCP<Epetra_Vector> X = rcp(new Epetra_Vector(epv->Map()));
X->PutScalar(0.0);
Epetra_LinearProblem epetraProblem(epA.get(), X.get(), epv.get());
AztecOO aztecSolver(epetraProblem);
aztecSolver.SetAztecOption(AZ_solver, AZ_gmres);
MueLu::EpetraOperator aztecPrec(H);
aztecSolver.SetPrecOperator(&aztecPrec);
int maxIts = 50;
double tol = 1e-8;
aztecSolver.Iterate(maxIts, tol);
}
success = true;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
#else
std::cout << "Epetra (and/or EpetraExt) are not available. Skip test." << std::endl;
return EXIT_SUCCESS;
#endif
}
int main(int argc, char *argv[]) {
typedef double MeshScalar;
typedef double BasisScalar;
typedef Tpetra::DefaultPlatform::DefaultPlatformType::NodeType Node;
typedef Teuchos::ScalarTraits<Scalar>::magnitudeType magnitudeType;
//double g_mean_exp = 1.906587e-01; // expected response mean
//double g_std_dev_exp = 8.680605e-02; // expected response std. dev.
//double g_tol = 1e-6; // tolerance on determining success
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::Array;
using Teuchos::ArrayRCP;
using Teuchos::ArrayView;
using Teuchos::ParameterList;
// Initialize MPI
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
#endif
// feenableexcept(FE_ALL_EXCEPT);
LocalOrdinal MyPID;
try {
// Create a communicator for Epetra objects
RCP<const Epetra_Comm> globalComm;
#ifdef HAVE_MPI
globalComm = rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
#else
globalComm = rcp(new Epetra_SerialComm);
#endif
MyPID = globalComm->MyPID();
// Setup command line options
Teuchos::CommandLineProcessor CLP;
CLP.setDocString(
"This example runs an interlaced stochastic Galerkin solvers.\n");
int n = 32;
CLP.setOption("num_mesh", &n, "Number of mesh points in each direction");
// multigrid specific options
int minAggSize = 1;
CLP.setOption("min_agg_size", &minAggSize, "multigrid aggregate size");
int smootherSweeps = 3;
CLP.setOption("smoother_sweeps", &smootherSweeps, "# multigrid smoother sweeps");
int plainAgg=1;
CLP.setOption("plain_aggregation", &plainAgg, "plain aggregation");
LocalOrdinal nsSize=-1;
CLP.setOption("nullspace_size", &nsSize, "nullspace dimension");
bool symmetric = false;
CLP.setOption("symmetric", "unsymmetric", &symmetric,
"Symmetric discretization");
int num_spatial_procs = -1;
CLP.setOption("num_spatial_procs", &num_spatial_procs, "Number of spatial processors (set -1 for all available procs)");
SG_RF randField = UNIFORM;
CLP.setOption("rand_field", &randField,
num_sg_rf, sg_rf_values, sg_rf_names,
"Random field type");
double mu = 0.2;
CLP.setOption("mean", &mu, "Mean");
double s = 0.1;
CLP.setOption("std_dev", &s, "Standard deviation");
int num_KL = 2;
CLP.setOption("num_kl", &num_KL, "Number of KL terms");
int order = 3;
CLP.setOption("order", &order, "Polynomial order");
bool normalize_basis = true;
CLP.setOption("normalize", "unnormalize", &normalize_basis,
"Normalize PC basis");
Krylov_Method solver_method = GMRES;
CLP.setOption("solver_method", &solver_method,
num_krylov_method, krylov_method_values, krylov_method_names,
"Krylov solver method");
SG_Prec prec_method = STOCHASTIC;
CLP.setOption("prec_method", &prec_method,
num_sg_prec, sg_prec_values, sg_prec_names,
"Preconditioner method");
SG_Div division_method = DIRECT;
CLP.setOption("division_method", &division_method,
num_sg_div, sg_div_values, sg_div_names,
"Stochastic division method");
SG_DivPrec divprec_method = NO;
CLP.setOption("divprec_method", &divprec_method,
num_sg_divprec, sg_divprec_values, sg_divprec_names,
"Preconditioner for division method");
Schur_option schur_option = diag;
CLP.setOption("schur_option", &schur_option,
num_schur_option, Schur_option_values, schur_option_names,
"Schur option");
Prec_option prec_option = whole;
CLP.setOption("prec_option", &prec_option,
num_prec_option, Prec_option_values, prec_option_names,
"Prec option");
double solver_tol = 1e-12;
CLP.setOption("solver_tol", &solver_tol, "Outer solver tolerance");
double div_tol = 1e-6;
CLP.setOption("div_tol", &div_tol, "Tolerance in Iterative Solver");
int prec_level = 1;
CLP.setOption("prec_level", &prec_level, "Level in Schur Complement Prec 0->Solve A0u0=g0 with division; 1->Form 1x1 Schur Complement");
int max_it_div = 50;
CLP.setOption("max_it_div", &max_it_div, "Maximum # of Iterations in Iterative Solver for Division");
bool equilibrate = true; //JJH 8/26/12 changing to true to match ETP example
CLP.setOption("equilibrate", "noequilibrate", &equilibrate,
"Equilibrate the linear system");
CLP.parse( argc, argv );
if (MyPID == 0) {
std::cout << "Summary of command line options:" << std::endl
<< "\tnum_mesh = " << n << std::endl
<< "\tsymmetric = " << symmetric << std::endl
<< "\tnum_spatial_procs = " << num_spatial_procs << std::endl
<< "\trand_field = " << sg_rf_names[randField]
<< std::endl
<< "\tmean = " << mu << std::endl
<< "\tstd_dev = " << s << std::endl
<< "\tnum_kl = " << num_KL << std::endl
<< "\torder = " << order << std::endl
<< "\tnormalize_basis = " << normalize_basis << std::endl
<< "\tsolver_method = " << krylov_method_names[solver_method] << std::endl
<< "\tprec_method = " << sg_prec_names[prec_method] << std::endl
<< "\tdivision_method = " << sg_div_names[division_method] << std::endl
<< "\tdiv_tol = " << div_tol << std::endl
<< "\tdiv_prec = " << sg_divprec_names[divprec_method] << std::endl
<< "\tprec_level = " << prec_level << std::endl
<< "\tmax_it_div = " << max_it_div << std::endl;
}
bool nonlinear_expansion = false;
if (randField == UNIFORM)
nonlinear_expansion = false;
else if (randField == LOGNORMAL)
nonlinear_expansion = true;
{
TEUCHOS_FUNC_TIME_MONITOR("Total PCE Calculation Time");
// Create Stochastic Galerkin basis and expansion
Teuchos::Array< RCP<const Stokhos::OneDOrthogPolyBasis<LocalOrdinal,BasisScalar> > > bases(num_KL);
for (LocalOrdinal i=0; i<num_KL; i++)
if (randField == UNIFORM)
bases[i] = rcp(new Stokhos::LegendreBasis<LocalOrdinal,BasisScalar>(order, normalize_basis));
else if (randField == LOGNORMAL)
bases[i] = rcp(new Stokhos::HermiteBasis<int,double>(order, normalize_basis));
RCP<const Stokhos::CompletePolynomialBasis<LocalOrdinal,BasisScalar> > basis =
rcp(new Stokhos::CompletePolynomialBasis<LocalOrdinal,BasisScalar>(bases, 1e-12));
LocalOrdinal sz = basis->size();
RCP<Stokhos::Sparse3Tensor<LocalOrdinal,BasisScalar> > Cijk =
basis->computeTripleProductTensor(sz);
RCP<const Stokhos::Quadrature<int,double> > quad =
rcp(new Stokhos::TensorProductQuadrature<int,double>(basis));
RCP<ParameterList> expn_params = Teuchos::rcp(new ParameterList);
if (division_method == MEAN_DIV) {
expn_params->set("Division Strategy", "Mean-Based");
expn_params->set("Use Quadrature for Division", false);
}
else if (division_method == DIRECT) {
expn_params->set("Division Strategy", "Dense Direct");
expn_params->set("Use Quadrature for Division", false);
}
else if (division_method == SPD_DIRECT) {
expn_params->set("Division Strategy", "SPD Dense Direct");
expn_params->set("Use Quadrature for Division", false);
}
else if (division_method == CGD) {
expn_params->set("Division Strategy", "CG");
expn_params->set("Use Quadrature for Division", false);
}
else if (division_method == QUAD) {
expn_params->set("Use Quadrature for Division", true);
}
if (divprec_method == NO)
expn_params->set("Prec Strategy", "None");
else if (divprec_method == DIAG)
expn_params->set("Prec Strategy", "Diag");
else if (divprec_method == JACOBI)
expn_params->set("Prec Strategy", "Jacobi");
else if (divprec_method == GS)
expn_params->set("Prec Strategy", "GS");
else if (divprec_method == SCHUR)
expn_params->set("Prec Strategy", "Schur");
if (schur_option == diag)
expn_params->set("Schur option", "diag");
else
expn_params->set("Schur option", "full");
if (prec_option == linear)
expn_params->set("Prec option", "linear");
if (equilibrate)
expn_params->set("Equilibrate", 1);
else
expn_params->set("Equilibrate", 0);
expn_params->set("Division Tolerance", div_tol);
expn_params->set("prec_iter", prec_level);
expn_params->set("max_it_div", max_it_div);
RCP<Stokhos::OrthogPolyExpansion<LocalOrdinal,BasisScalar> > expansion =
rcp(new Stokhos::QuadOrthogPolyExpansion<LocalOrdinal,BasisScalar>(
basis, Cijk, quad, expn_params));
if (MyPID == 0)
std::cout << "Stochastic Galerkin expansion size = " << sz << std::endl;
// Create stochastic parallel distribution
ParameterList parallelParams;
parallelParams.set("Number of Spatial Processors", num_spatial_procs);
// parallelParams.set("Rebalance Stochastic Graph", true);
// Teuchos::ParameterList& isorropia_params =
// parallelParams.sublist("Isorropia");
// isorropia_params.set("Balance objective", "nonzeros");
RCP<Stokhos::ParallelData> sg_parallel_data =
rcp(new Stokhos::ParallelData(basis, Cijk, globalComm, parallelParams));
RCP<const EpetraExt::MultiComm> sg_comm =
sg_parallel_data->getMultiComm();
RCP<const Epetra_Comm> app_comm =
sg_parallel_data->getSpatialComm();
// Create Teuchos::Comm from Epetra_Comm
RCP< Teuchos::Comm<int> > teuchos_app_comm;
#ifdef HAVE_MPI
RCP<const Epetra_MpiComm> app_mpi_comm =
Teuchos::rcp_dynamic_cast<const Epetra_MpiComm>(app_comm);
RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > raw_mpi_comm =
Teuchos::opaqueWrapper(app_mpi_comm->Comm());
teuchos_app_comm = rcp(new Teuchos::MpiComm<int>(raw_mpi_comm));
#else
teuchos_app_comm = rcp(new Teuchos::SerialComm<int>());
#endif
// Create application
typedef twoD_diffusion_problem<Scalar,MeshScalar,BasisScalar,LocalOrdinal,GlobalOrdinal,Node> problem_type;
RCP<problem_type> model =
rcp(new problem_type(teuchos_app_comm, n, num_KL, s, mu,
nonlinear_expansion, symmetric));
// Create vectors and operators
typedef problem_type::Tpetra_Vector Tpetra_Vector;
typedef problem_type::Tpetra_CrsMatrix Tpetra_CrsMatrix;
typedef Tpetra::MatrixMarket::Writer<Tpetra_CrsMatrix> Writer;
//Xpetra matrices
typedef Xpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_CrsMatrix;
typedef Xpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> Xpetra_MultiVector;
typedef Xpetra::MultiVectorFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node> Xpetra_MultiVectorFactory;
typedef Xpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_Operator;
typedef Xpetra::TpetraCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_TpetraCrsMatrix;
typedef Xpetra::CrsOperator<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Xpetra_CrsOperator;
typedef Belos::MueLuOp<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> Belos_MueLuOperator;
//MueLu typedefs
typedef MueLu::Hierarchy<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> MueLu_Hierarchy;
typedef MueLu::SmootherPrototype<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> SmootherPrototype;
typedef MueLu::TrilinosSmoother<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> TrilinosSmoother;
typedef MueLu::SmootherFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> SmootherFactory;
typedef MueLu::FactoryManager<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> FactoryManager;
RCP<Tpetra_Vector> p = Tpetra::createVector<Scalar>(model->get_p_map(0));
RCP<Tpetra_Vector> x = Tpetra::createVector<Scalar>(model->get_x_map());
x->putScalar(0.0);
RCP<Tpetra_Vector> f = Tpetra::createVector<Scalar>(model->get_f_map());
RCP<Tpetra_Vector> dx = Tpetra::createVector<Scalar>(model->get_x_map());
RCP<Tpetra_CrsMatrix> J = model->create_W();
RCP<Tpetra_CrsMatrix> J0;
if (prec_method == MEAN)
J0 = model->create_W();
// Set PCE expansion of p
p->putScalar(0.0);
ArrayRCP<Scalar> p_view = p->get1dViewNonConst();
for (ArrayRCP<Scalar>::size_type i=0; i<p_view.size(); i++) {
p_view[i].reset(expansion);
p_view[i].copyForWrite();
}
Array<double> point(num_KL, 1.0);
Array<double> basis_vals(sz);
basis->evaluateBases(point, basis_vals);
if (order > 0) {
for (int i=0; i<num_KL; i++) {
p_view[i].term(i,1) = 1.0 / basis_vals[i+1];
}
}
// Create preconditioner
typedef Ifpack2::Preconditioner<Scalar,LocalOrdinal,GlobalOrdinal,Node> Tprec;
RCP<Belos_MueLuOperator> M;
RCP<MueLu_Hierarchy> H;
RCP<Xpetra_CrsMatrix> xcrsJ = rcp(new Xpetra_TpetraCrsMatrix(J));
RCP<Xpetra_Operator> xopJ = rcp(new Xpetra_CrsOperator(xcrsJ));
if (prec_method != NONE) {
ParameterList precParams;
std::string prec_name = "RILUK";
precParams.set("fact: iluk level-of-fill", 1);
precParams.set("fact: iluk level-of-overlap", 0);
//Ifpack2::Factory factory;
RCP<Xpetra_Operator> xopJ0;
if (prec_method == MEAN) {
RCP<Xpetra_CrsMatrix> xcrsJ0 = rcp(new Xpetra_TpetraCrsMatrix(J0));
xopJ0 = rcp(new Xpetra_CrsOperator(xcrsJ0));
//M = factory.create<Tpetra_CrsMatrix>(prec_name, J0);
} else if (prec_method == STOCHASTIC) {
xopJ0 = xopJ;
//M = factory.create<Tpetra_CrsMatrix>(prec_name, J);
}
H = rcp(new MueLu_Hierarchy(xopJ0));
M = rcp(new Belos_MueLuOperator(H));
//M->setParameters(precParams);
if (nsSize!=-1) sz=nsSize;
RCP<Xpetra_MultiVector> Z = Xpetra_MultiVectorFactory::Build(xcrsJ->getDomainMap(), sz);
size_t n = Z->getLocalLength();
for (LocalOrdinal j=0; j<sz; ++j) {
ArrayRCP<Scalar> col = Z->getDataNonConst(j);
for (size_t i=0; i<n; ++i) {
col[i].reset(expansion);
col[i].copyForWrite();
col[i].fastAccessCoeff(j) = 1.0;
}
}
H->GetLevel(0)->Set("Nullspace", Z);
//RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
//fos->setOutputToRootOnly(-1);
//Z->describe(*fos);
}
// Evaluate model
model->computeResidual(*x, *p, *f);
model->computeJacobian(*x, *p, *J);
// Compute mean for mean-based preconditioner
if (prec_method == MEAN) {
size_t nrows = J->getNodeNumRows();
ArrayView<const LocalOrdinal> indices;
ArrayView<const Scalar> values;
J0->resumeFill();
for (size_t i=0; i<nrows; i++) {
J->getLocalRowView(i, indices, values);
Array<Scalar> values0(values.size());
for (LocalOrdinal j=0; j<values.size(); j++)
values0[j] = values[j].coeff(0);
J0->replaceLocalValues(i, indices, values0);
}
J0->fillComplete();
}
// compute preconditioner
if (prec_method != NONE) {
//M->initialize();
//M->compute();
//override MueLu defaults via factory manager
RCP<FactoryManager> fm = rcp( new FactoryManager() );;
//smoother
ParameterList smootherParamList;
/*
smootherParamList.set("chebyshev: degree", smootherSweeps);
smootherParamList.set("chebyshev: ratio eigenvalue", (double) 20);
smootherParamList.set("chebyshev: max eigenvalue", (double) -1.0);
smootherParamList.set("chebyshev: min eigenvalue", (double) 1.0);
smootherParamList.set("chebyshev: zero starting solution", true);
RCP<SmootherPrototype> smooPrototype = rcp( new TrilinosSmoother("CHEBYSHEV", smootherParamList) );
*/
smootherParamList.set("relaxation: sweeps", smootherSweeps);
smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
RCP<SmootherPrototype> smooPrototype = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );
RCP<SmootherFactory> smooFact = rcp( new SmootherFactory(smooPrototype) );
fm->SetFactory("Smoother", smooFact);
// coarse level solve
ParameterList coarseParamList;
coarseParamList.set("fact: level-of-fill", 0);
RCP<SmootherPrototype> coarsePrototype = rcp( new TrilinosSmoother("ILUT", coarseParamList) );
RCP<SmootherFactory> coarseSolverFact = rcp( new SmootherFactory(coarsePrototype, Teuchos::null) );
fm->SetFactory("CoarseSolver", coarseSolverFact);
//allow for larger aggregates
typedef MueLu::UCAggregationFactory<LocalOrdinal,GlobalOrdinal,Node,LocalMatOps>
MueLu_UCAggregationFactory;
RCP<MueLu_UCAggregationFactory> aggFact = rcp(new MueLu_UCAggregationFactory());
aggFact->SetMinNodesPerAggregate(minAggSize);
fm->SetFactory("Aggregates", aggFact);
//turn off damping
typedef MueLu::SaPFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> MueLu_SaPFactory;
if (plainAgg) {
RCP<MueLu_SaPFactory> sapFactory = rcp(new MueLu_SaPFactory);
sapFactory->SetDampingFactor( (Scalar) 0.0 );
fm->SetFactory("P", sapFactory);
}
H->Setup(*fm);
}
// Setup Belos solver
RCP<ParameterList> belosParams = rcp(new ParameterList);
belosParams->set("Flexible Gmres", false);
belosParams->set("Num Blocks", 500);//20
belosParams->set("Convergence Tolerance", solver_tol);
belosParams->set("Maximum Iterations", 1000);
belosParams->set("Verbosity", 33);
belosParams->set("Output Style", 1);
belosParams->set("Output Frequency", 1);
typedef Tpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> MV;
typedef Belos::OperatorT<Tpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> > OP;
typedef Belos::OperatorTraits<Scalar,MV,OP> BOPT;
typedef Belos::MultiVecTraits<Scalar,MV> BMVT;
typedef Belos::MultiVecTraits<double,MV> BTMVT;
typedef Belos::LinearProblem<double,MV,OP> BLinProb;
typedef Belos::XpetraOp<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> BXpetraOp;
RCP<OP> belosJ = rcp(new BXpetraOp(xopJ)); // Turns an Xpetra::Operator object into a Belos operator
RCP< BLinProb > problem = rcp(new BLinProb(belosJ, dx, f));
if (prec_method != NONE)
problem->setRightPrec(M);
problem->setProblem();
RCP<Belos::SolverManager<double,MV,OP> > solver;
if (solver_method == CG)
solver = rcp(new Belos::PseudoBlockCGSolMgr<double,MV,OP>(problem, belosParams));
else if (solver_method == GMRES)
solver = rcp(new Belos::BlockGmresSolMgr<double,MV,OP>(problem, belosParams));
// Print initial residual norm
std::vector<double> norm_f(1);
//BMVT::MvNorm(*f, norm_f);
BTMVT::MvNorm(*f, norm_f);
if (MyPID == 0)
std::cout << "\nInitial residual norm = " << norm_f[0] << std::endl;
// Solve linear system
Belos::ReturnType ret = solver->solve();
if (MyPID == 0) {
if (ret == Belos::Converged)
std::cout << "Solver converged!" << std::endl;
else
std::cout << "Solver failed to converge!" << std::endl;
}
// Update x
x->update(-1.0, *dx, 1.0);
Writer::writeDenseFile("stochastic_solution.mm", x);
// Compute new residual & response function
RCP<Tpetra_Vector> g = Tpetra::createVector<Scalar>(model->get_g_map(0));
f->putScalar(0.0);
model->computeResidual(*x, *p, *f);
model->computeResponse(*x, *p, *g);
// Print final residual norm
//BMVT::MvNorm(*f, norm_f);
BTMVT::MvNorm(*f, norm_f);
if (MyPID == 0)
std::cout << "\nFinal residual norm = " << norm_f[0] << std::endl;
// Print response
std::cout << "\nResponse = " << std::endl;
//Writer::writeDense(std::cout, g);
Writer::writeDenseFile("stochastic_residual.mm", f);
/*
double g_mean = g->get1dView()[0].mean();
double g_std_dev = g->get1dView()[0].standard_deviation();
std::cout << "g mean = " << g_mean << std::endl;
std::cout << "g std_dev = " << g_std_dev << std::endl;
bool passed = false;
if (norm_f[0] < 1.0e-10 &&
std::abs(g_mean-g_mean_exp) < g_tol &&
std::abs(g_std_dev - g_std_dev_exp) < g_tol)
passed = true;
if (MyPID == 0) {
if (passed)
std::cout << "Example Passed!" << std::endl;
else{
std::cout << "Example Failed!" << std::endl;
std::cout << "expected g_mean = "<< g_mean_exp << std::endl;
std::cout << "expected g_std_dev = "<< g_std_dev_exp << std::endl;
}
}
*/
}
Teuchos::TimeMonitor::summarize(std::cout);
Teuchos::TimeMonitor::zeroOutTimers();
}
catch (std::exception& e) {
std::cout << e.what() << std::endl;
}
catch (string& s) {
std::cout << s << std::endl;
}
catch (char *s) {
std::cout << s << std::endl;
}
catch (...) {
std::cout << "Caught unknown exception!" <<std:: endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
using Teuchos::ParameterList;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100, nz = 100;
Galeri::Xpetra::Parameters<GO> galeriParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file [default = 'scalingTest.xml']");
bool printTimings = true; clp.setOption("timings", "notimings", &printTimings, "print timings to screen");
int writeMatricesOPT = -2; clp.setOption("write", &writeMatricesOPT, "write matrices to file (-1 means all; i>=0 means level i)");
std::string dsolveType = "cg", solveType; clp.setOption("solver", &dsolveType, "solve type: (none | cg | gmres | standalone)");
double dtol = 1e-12, tol; clp.setOption("tol", &dtol, "solver convergence tolerance");
std::string mapFile; clp.setOption("map", &mapFile, "map data file");
std::string matrixFile; clp.setOption("matrix", &matrixFile, "matrix data file");
std::string coordFile; clp.setOption("coords", &coordFile, "coordinates data file");
int numRebuilds = 0; clp.setOption("rebuild", &numRebuilds, "#times to rebuild hierarchy");
int maxIts = 200; clp.setOption("its", &maxIts, "maximum number of solver iterations");
bool scaleResidualHistory = true; clp.setOption("scale", "noscale", &scaleResidualHistory, "scaled Krylov residual history");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<ParameterList>(¶mList), *comm);
bool isDriver = paramList.isSublist("Run1");
if (isDriver) {
// update galeriParameters with the values from the XML file
ParameterList& realParams = galeriParameters.GetParameterList();
for (ParameterList::ConstIterator it = realParams.begin(); it != realParams.end(); it++) {
const std::string& name = realParams.name(it);
if (paramList.isParameter(name))
realParams.setEntry(name, paramList.getEntry(name));
}
}
// Retrieve matrix parameters (they may have been changed on the command line)
// [for instance, if we changed matrix type from 2D to 3D we need to update nz]
ParameterList galeriList = galeriParameters.GetParameterList();
// =========================================================================
// Problem construction
// =========================================================================
std::ostringstream galeriStream;
comm->barrier();
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
RCP<TimeMonitor> tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<Matrix> A;
RCP<const Map> map;
RCP<MultiVector> coordinates;
RCP<MultiVector> nullspace;
if (matrixFile.empty()) {
galeriStream << "========================================================\n" << xpetraParameters << galeriParameters;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
std::string matrixType = galeriParameters.GetMatrixType();
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriList);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriList);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriList);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
galeriStream << "Processor subdomains in x direction: " << galeriList.get<int>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriList.get<int>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriList.get<int>("mz") << std::endl
<< "========================================================" << std::endl;
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") {
// Our default test case for elasticity: all boundaries of a square/cube have Neumann b.c. except left which has Dirichlet
galeriList.set("right boundary" , "Neumann");
galeriList.set("bottom boundary", "Neumann");
galeriList.set("top boundary" , "Neumann");
galeriList.set("front boundary" , "Neumann");
galeriList.set("back boundary" , "Neumann");
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(galeriParameters.GetMatrixType(), map, galeriList);
A = Pr->BuildMatrix();
nullspace = MultiVectorFactory::Build(map, 1);
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((galeriParameters.GetMatrixType() == "Elasticity2D") ? 2 : 3);
} else {
nullspace->putScalar(one);
}
} else {
if (!mapFile.empty())
map = Utils2::ReadMap(mapFile, xpetraParameters.GetLib(), comm);
comm->barrier();
if (lib == Xpetra::UseEpetra) {
A = Utils::Read(matrixFile, map);
} else {
// Tpetra matrix reader is still broken, so instead we read in
// a matrix in a binary format and then redistribute it
const bool binaryFormat = true;
A = Utils::Read(matrixFile, lib, comm, binaryFormat);
RCP<Matrix> newMatrix = MatrixFactory::Build(map, 1);
RCP<Import> importer = ImportFactory::Build(A->getRowMap(), map);
newMatrix->doImport(*A, *importer, Xpetra::INSERT);
newMatrix->fillComplete();
A.swap(newMatrix);
}
comm->barrier();
if (!coordFile.empty())
coordinates = Utils2::ReadMultiVector(coordFile, map);
nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(one);
}
comm->barrier();
tm = Teuchos::null;
galeriStream << "Galeri complete.\n========================================================" << std::endl;
int numReruns = 1;
if (paramList.isParameter("number of reruns"))
numReruns = paramList.get<int>("number of reruns");
const bool mustAlreadyExist = true;
for (int rerunCount = 1; rerunCount <= numReruns; rerunCount++) {
ParameterList mueluList, runList;
bool stop = false;
if (isDriver) {
runList = paramList.sublist("Run1", mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} else {
mueluList = paramList;
stop = true;
}
int runCount = 1;
do {
A->SetMaxEigenvalueEstimate(-one);
solveType = dsolveType;
tol = dtol;
int savedOut = -1;
FILE* openedOut = NULL;
if (isDriver) {
if (runList.isParameter("filename")) {
// Redirect all output into a filename We have to redirect all output,
// including printf's, therefore we cannot simply replace C++ cout
// buffers, and have to use heavy machinary (dup2)
std::string filename = runList.get<std::string>("filename");
if (numReruns > 1)
filename += "_run" + MueLu::toString(rerunCount);
filename += (lib == Xpetra::UseEpetra ? ".epetra" : ".tpetra");
savedOut = dup(STDOUT_FILENO);
openedOut = fopen(filename.c_str(), "w");
dup2(fileno(openedOut), STDOUT_FILENO);
}
if (runList.isParameter("solver")) solveType = runList.get<std::string>("solver");
if (runList.isParameter("tol")) tol = runList.get<double> ("tol");
}
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
fancyout << galeriStream.str();
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
RCP<HierarchyManager> mueLuFactory = rcp(new ParameterListInterpreter(mueluList));
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
RCP<Hierarchy> H;
for (int i = 0; i <= numRebuilds; i++) {
A->SetMaxEigenvalueEstimate(-one);
H = mueLuFactory->CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
if (!coordinates.is_null())
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory->SetupHierarchy(*H);
}
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
{
// we set seed for reproducibility
Utils::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
tm = Teuchos::null;
if (writeMatricesOPT > -2) {
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3.5 - Matrix output")));
H->Write(writeMatricesOPT, writeMatricesOPT);
tm = Teuchos::null;
}
comm->barrier();
if (solveType == "none") {
// Do not perform a solve
} else if (solveType == "standalone") {
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Fixed Point Solve")));
H->IsPreconditioner(false);
H->Iterate(*B, *X, maxIts);
} else if (solveType == "cg" || solveType == "gmres") {
#ifdef HAVE_MUELU_BELOS
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp <SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
fancyout << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
if (!scaleResidualHistory)
belosList.set("Implicit Residual Scaling", "None");
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg") {
solver = rcp(new Belos::PseudoBlockCGSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
} else if (solveType == "gmres") {
solver = rcp(new Belos::BlockGmresSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
}
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
fancyout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
} catch(...) {
fancyout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged)
fancyout << std::endl << "ERROR: Belos did not converge! " << std::endl;
else
fancyout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif //ifdef HAVE_MUELU_BELOS
} else {
throw MueLu::Exceptions::RuntimeError("Unknown solver type: \"" + solveType + "\"");
}
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union);
TimeMonitor::clearCounters();
if (isDriver) {
if (openedOut != NULL) {
dup2(savedOut, STDOUT_FILENO);
fclose(openedOut);
openedOut = NULL;
}
try {
runList = paramList.sublist("Run" + MueLu::toString(++runCount), mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} catch (std::exception) {
stop = true;
}
}
} while (stop == false);
}
return 0;
} //main
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP; using Teuchos::rcp;
using Teuchos::TimeMonitor;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole);
bool success = false;
bool verbose = true;
try {
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
Teuchos::CommandLineProcessor clp(false);
std::string xmlFileName = "TwoBillion.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file");
int num_per_proc = 1000; clp.setOption("dpc", &num_per_proc, "DOFs per core");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
int NumProcs = comm->getSize();
int MyPID = comm->getRank();
if(!MyPID) printf("TwoBillion: Running Test\n");
const long long FIRST_GID = 3000000000L;
// const long long FIRST_GID = 0L;
const long long IndexBase = 0L;
// const long long IndexBase = 3000000000L;
global_size_t NumGlobalElements = NumProcs*num_per_proc;
// Create Map w/ GIDs starting at > 2 billion
RCP<const Map> map;
RCP<CrsMatrix> Acrs;
Teuchos::Array<GlobalOrdinal> mygids(num_per_proc);
for(int i=0; i<num_per_proc; i++)
mygids[i] = FIRST_GID + MyPID*num_per_proc + i;
for (int i=0; i<NumProcs; ++i) {
if (i==MyPID)
std::cout << "pid " << i << " : 1st GID = " << mygids[0] << std::endl;
}
//for(int i=0;i<num_per_proc;i++)
// printf("[%d] mygids[%d] = %lld\n",MyPID,i,mygids[i]);
map = MapFactory::Build(Xpetra::UseTpetra, Teuchos::OrdinalTraits<global_size_t>::invalid(),mygids(),IndexBase,comm);
// RCP<Teuchos::FancyOStream> fox = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
// fox->setOutputToRootOnly(-1);
// map->describe(*fox,Teuchos::VERB_EXTREME);
// Create 1D Laplacian w/ GIDs starting at > 2 billion
Teuchos::Array<Scalar> myvals(3);
Teuchos::Array<GlobalOrdinal> mycols(3);
Teuchos::ArrayView<Scalar> ValView;
Teuchos::ArrayView<GlobalOrdinal> ColView;
Acrs = CrsMatrixFactory::Build(map,3);
for(int i=0; i<num_per_proc; i++) {
if(mygids[i]==FIRST_GID ) {
mycols[0] = mygids[i]; myvals[0] = 2;
mycols[1] = mygids[i]+1; myvals[1] = -1;
ValView=myvals.view(0,2);
ColView=mycols.view(0,2);
// printf("[%d %lld] cols %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1]);
}
else if(mygids[i] == FIRST_GID + (long long) NumGlobalElements - 1){
mycols[0] = mygids[i]-1; myvals[0] = -1;
mycols[1] = mygids[i]; myvals[1] = 2;
ValView=myvals.view(0,2);
ColView=mycols.view(0,2);
// printf("[%d %lld] cols %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1]);
}
else {
mycols[0] = mygids[i]-1; myvals[0] = -1;
mycols[1] = mygids[i]; myvals[1] = -2;
mycols[2] = mygids[i]+1; myvals[1] = -1;
ValView=myvals();
ColView=mycols();
// printf("[%d %lld] cols %lld %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1],mycols[2]);
}
Acrs->insertGlobalValues(mygids[i],ColView,ValView);
}
Acrs->fillComplete();
RCP<Matrix> A = rcp(new CrsMatrixWrap(Acrs));
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
RCP<MultiVector> coordinates = MultiVectorFactory::Build(map, 1);
Teuchos::ArrayRCP<Scalar> coordVals = coordinates->getDataNonConst(0);
double h = 1.0 / NumGlobalElements;
for (LocalOrdinal i=0; i<num_per_proc; ++i)
coordVals[i] = MyPID*num_per_proc*h + i*h;
coordVals = Teuchos::null;
Teuchos::ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
RCP<HierarchyManager> mueLuFactory;
mueLuFactory = rcp(new ParameterListInterpreter(xmlFileName, *comm));
RCP<Hierarchy> H;
H = mueLuFactory->CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory->SetupHierarchy(*H);
//
//
// SOLVE
//
//
// Define X, B
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
X->setSeed(846930886);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
B->norm2(norms);
B->scale(1.0/norms[0]);
//
// Use AMG as a preconditioner in Belos
//
#ifdef HAVE_MUELU_BELOS
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO>(A)); // Turns a Xpetra::Operator object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 100;
double optTol = 1e-8;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", optTol); // Relative convergence tolerance requested
//belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
if (comm->getRank() == 0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
if (solver->getNumIters() > 6) {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
return(EXIT_FAILURE);
}
// Compute actual residuals.
int numrhs = 1;
std::vector<double> actual_resids( numrhs ); //TODO: double?
std::vector<double> rhs_norm( numrhs );
RCP<MultiVector> resid = MultiVectorFactory::Build(map, numrhs);
typedef Belos::OperatorTraits<SC, MV, OP> OPT;
typedef Belos::MultiVecTraits<SC, MV> MVT;
OPT::Apply( *belosOp, *X, *resid );
MVT::MvAddMv( -1.0, *resid, 1.0, *B, *resid );
MVT::MvNorm( *resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
*out<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for ( int i = 0; i<numrhs; i++) {
double actRes = actual_resids[i]/rhs_norm[i];
*out<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
//if (actRes > tol) { badRes = true; }
}
} //try
catch(...) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
success = (ret == Belos::Converged);
// Check convergence
if (success) {
if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
} else {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
}
#endif // HAVE_MUELU_BELOS
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
Teuchos::RCP<Vector> runExample(std::vector<size_t> stridingInfo, LocalOrdinal stridedBlockId, GlobalOrdinal offset) {
using Teuchos::RCP;
using Teuchos::rcp;
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// Timing
Teuchos::Time myTime("global");
Teuchos::TimeMonitor Mt(myTime);
#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
#endif
// custom parameters
LO maxLevels = 4;
GO maxCoarseSize=1; //FIXME clp doesn't like long long int
std::string aggOrdering = "natural";
int minPerAgg=3;
int maxNbrAlreadySelected=0;
////////////////////////////////////////////////////////////////////////////////////////
// prepare redistribution of matrix (parallelization)
int globalNumDofs = 7020;
int nProcs = comm->getSize();
int nDofsPerNode = 2;
int nLocalDofs = (int) globalNumDofs / nProcs;
nLocalDofs = nLocalDofs - (nLocalDofs % nDofsPerNode);
int nCumulatedDofs = 0;
sumAll(comm,nLocalDofs, nCumulatedDofs);
if(comm->getRank() == nProcs-1) {
nLocalDofs += globalNumDofs - nCumulatedDofs;
}
std::cout << "PROC: " << comm->getRank() << " numLocalDofs=" << nLocalDofs << std::endl;
////////////////////////////////////////////////////////////////////////////////////////
// read in problem
Epetra_Map emap (globalNumDofs, nLocalDofs, 0, *Xpetra::toEpetra(comm));
Epetra_CrsMatrix * ptrA = 0;
Epetra_Vector * ptrf = 0;
Epetra_MultiVector* ptrNS = 0;
std::cout << "Reading matrix market file" << std::endl;
EpetraExt::MatrixMarketFileToCrsMatrix("stru2d_A.txt",emap,emap,emap,ptrA);
EpetraExt::MatrixMarketFileToVector("stru2d_b.txt",emap,ptrf);
EpetraExt::MatrixMarketFileToMultiVector( "stru2d_ns.txt", emap, ptrNS);
RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
RCP<Epetra_Vector> epv = Teuchos::rcp(ptrf);
RCP<Epetra_MultiVector> epNS = Teuchos::rcp(ptrNS);
////////////////////////////////////////////
// Epetra_CrsMatrix -> Xpetra::Matrix
RCP<CrsMatrix> exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(epA));
RCP<CrsMatrixWrap> crsOp = Teuchos::rcp(new CrsMatrixWrap(exA));
RCP<Matrix> Op = Teuchos::rcp_dynamic_cast<Matrix>(crsOp);
Op->SetFixedBlockSize(nDofsPerNode);
// Epetra_Vector -> Xpetra::Vector
RCP<Vector> xRhs = Teuchos::rcp(new Xpetra::EpetraVector(epv));
RCP<MultiVector> xNS = Teuchos::rcp(new Xpetra::EpetraMultiVector(epNS));
// Epetra_Map -> Xpetra::Map
const RCP< const Map> map = Xpetra::toXpetra(emap);
////////////////////////////////////////////
// create new MueLu hierarchy
RCP<Hierarchy> H = rcp ( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
H->SetMaxCoarseSize(maxCoarseSize);
// build finest Level
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op);
Finest->Set("Nullspace",xNS);
// prepare CoalesceDropFactory
RCP<CoalesceDropFactory> dropFact = rcp(new CoalesceDropFactory());
//dropFact->SetVariableBlockSize();
// prepare aggregation strategy
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetFactory("Graph", dropFact);
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min DOFs per aggregate : " << minPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << maxNbrAlreadySelected << std::endl;
CoupledAggFact->SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D
CoupledAggFact->SetMaxNeighAlreadySelected(maxNbrAlreadySelected);
std::transform(aggOrdering.begin(), aggOrdering.end(), aggOrdering.begin(), ::tolower);
if (aggOrdering == "natural") {
*out << "aggregate ordering : NATURAL" << std::endl;
CoupledAggFact->SetOrdering(MueLu::AggOptions::NATURAL);
} else if (aggOrdering == "random") {
*out << "aggregate ordering : RANDOM" << std::endl;
CoupledAggFact->SetOrdering(MueLu::AggOptions::RANDOM);
} else if (aggOrdering == "graph") {
*out << "aggregate ordering : GRAPH" << std::endl;
CoupledAggFact->SetOrdering(MueLu::AggOptions::GRAPH);
} else {
std::string msg = "main: bad aggregation option """ + aggOrdering + """.";
throw(MueLu::Exceptions::RuntimeError(msg));
}
CoupledAggFact->SetPhase3AggCreation(0.5);
*out << "=============================================================================" << std::endl;
// build transfer operators
RCP<TentativePFactory> TentPFact = rcp(new TentativePFactory());
/*TentPFact->setStridingData(stridingInfo);
TentPFact->setStridedBlockId(stridedBlockId);
TentPFact->setDomainMapOffset(offset);*/
RCP<CoarseMapFactory> coarseMapFact = Teuchos::rcp(new CoarseMapFactory());
coarseMapFact->setStridingData(stridingInfo);
coarseMapFact->setStridedBlockId(stridedBlockId);
coarseMapFact->setDomainMapOffset(offset);
RCP<SaPFactory> Pfact = rcp( new SaPFactory() );
//RCP<PgPFactory> Pfact = rcp( new PgPFactory() );
//RCP<TentativePFactory> Pfact = rcp( new TentativePFactory() );
RCP<Factory> Rfact = rcp( new TransPFactory() );
//RCP<Factory> Rfact = rcp( new GenericRFactory() );
// RAP Factory
RCP<RAPFactory> Acfact = rcp( new RAPFactory() );
Acfact->setVerbLevel(Teuchos::VERB_HIGH);
// register aggregation export factory in RAPFactory
//RCP<MueLu::AggregationExportFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node, LocalMatOps> > aggExpFact = rcp(new MueLu::AggregationExportFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node, LocalMatOps>());
//aggExpFact->SetParameter("Output filename","aggs_level%LEVELID_proc%PROCID.out");
//Acfact->AddTransferFactory(aggExpFact);
// build level smoothers
RCP<SmootherPrototype> smooProto;
std::string ifpackType;
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) 1);
ifpackList.set("relaxation: damping factor", (SC) 1.0); // 0.7
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
smooProto = Teuchos::rcp( new TrilinosSmoother(ifpackType, ifpackList) );
RCP<SmootherFactory> SmooFact;
if (maxLevels > 1)
SmooFact = rcp( new SmootherFactory(smooProto) );
// create coarsest smoother
RCP<SmootherPrototype> coarsestSmooProto;
std::string type = "";
Teuchos::ParameterList coarsestSmooList;
#if defined(HAVE_AMESOS_SUPERLU)
coarsestSmooProto = Teuchos::rcp( new DirectSolver("Superlu", coarsestSmooList) );
#else
coarsestSmooProto = Teuchos::rcp( new DirectSolver("Klu", coarsestSmooList) );
#endif
RCP<SmootherFactory> coarsestSmooFact;
coarsestSmooFact = rcp(new SmootherFactory(coarsestSmooProto, Teuchos::null));
FactoryManager M;
//M.SetFactory("Graph", dropFact);
//M.SetFactory("UnAmalgamationInfo", dropFact);
M.SetFactory("Aggregates", CoupledAggFact);
M.SetFactory("Ptent", TentPFact);
M.SetFactory("P", Pfact);
M.SetFactory("R", Rfact);
M.SetFactory("A", Acfact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", coarsestSmooFact);
M.SetFactory("CoarseMap", coarseMapFact);
H->Setup(M, 0, maxLevels);
RCP<Vector> xLsg = VectorFactory::Build(map);
// Use AMG directly as an iterative method
{
xLsg->putScalar( (SC) 0.0);
H->Iterate(*xRhs,10,*xLsg);
//xLsg->describe(*out,Teuchos::VERB_EXTREME);
}
//
// Solve Ax = b using AMG as a preconditioner in AztecOO
//
{
RCP<Epetra_Vector> X = rcp(new Epetra_Vector(epv->Map()));
X->PutScalar(0.0);
Epetra_LinearProblem epetraProblem(epA.get(), X.get(), epv.get());
AztecOO aztecSolver(epetraProblem);
aztecSolver.SetAztecOption(AZ_solver, AZ_cg);
MueLu::EpetraOperator aztecPrec(H);
aztecSolver.SetPrecOperator(&aztecPrec);
int maxIts = 50;
double tol = 1e-8;
aztecSolver.Iterate(maxIts, tol);
}
return xLsg;
}
TEUCHOS_UNIT_TEST(GenericRFactory, SymmetricProblem)
{
out << "version: " << MueLu::Version() << std::endl;
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// generate problem
LO maxLevels = 3;
LO nEle = 63;
const RCP<const Map> map = MapFactory::Build(TestHelpers::Parameters::getLib(), nEle, 0, comm);
Teuchos::ParameterList matrixParameters;
matrixParameters.set("nx",nEle);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Map, CrsMatrixWrap, MultiVector>("Laplace1D", map, matrixParameters);
RCP<Matrix> Op = Pr->BuildMatrix();
// build nullspace
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
out << "||NS|| = " << norms[0] << std::endl;
// fill hierarchy
RCP<Hierarchy> H = rcp( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
RCP<Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op); // set fine level matrix
Finest->Set("Nullspace",nullSpace); // set null space information for finest level
// define transfer operators
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetMinNodesPerAggregate(3);
CoupledAggFact->SetMaxNeighAlreadySelected(0);
CoupledAggFact->SetOrdering("natural");
CoupledAggFact->SetPhase3AggCreation(0.5);
RCP<SaPFactory> Pfact = rcp( new SaPFactory());
RCP<Factory> Rfact = rcp( new GenericRFactory() );
H->SetMaxCoarseSize(1);
// setup smoothers
Teuchos::ParameterList smootherParamList;
smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
smootherParamList.set("relaxation: sweeps", (LO) 1);
smootherParamList.set("relaxation: damping factor", (SC) 1.0);
RCP<SmootherPrototype> smooProto = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );
RCP<SmootherFactory> SmooFact = rcp( new SmootherFactory(smooProto) );
//Acfact->setVerbLevel(Teuchos::VERB_HIGH);
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(smooProto, Teuchos::null));
FactoryManager M;
M.SetFactory("P", Pfact);
M.SetFactory("R", Rfact);
M.SetFactory("Aggregates", CoupledAggFact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", coarseSolveFact);
H->Setup(M, 0, maxLevels);
RCP<Level> coarseLevel = H->GetLevel(1);
RCP<Matrix> P1 = coarseLevel->Get< RCP<Matrix> >("P");
RCP<Matrix> R1 = coarseLevel->Get< RCP<Matrix> >("R");
RCP<Level> coarseLevel2 = H->GetLevel(2);
RCP<Matrix> P2 = coarseLevel2->Get< RCP<Matrix> >("P");
RCP<Matrix> R2 = coarseLevel2->Get< RCP<Matrix> >("R");
TEST_EQUALITY(Finest->IsAvailable("PreSmoother"), true);
TEST_EQUALITY(Finest->IsAvailable("PostSmoother"), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PreSmoother"), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PostSmoother"), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PreSmoother"), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PostSmoother"), false);
// test some basic multgrid data
TEST_EQUALITY(P1->getGlobalNumEntries(), R1->getGlobalNumEntries());
TEST_EQUALITY(P1->getGlobalNumRows(), R1->getGlobalNumCols());
TEST_EQUALITY(P1->getGlobalNumCols(), R1->getGlobalNumRows());
TEST_EQUALITY(P2->getGlobalNumEntries(), R2->getGlobalNumEntries());
TEST_EQUALITY(P2->getGlobalNumRows(), R2->getGlobalNumCols());
TEST_EQUALITY(P2->getGlobalNumCols(), R2->getGlobalNumRows());
//RCP<Teuchos::FancyOStream> fos = getFancyOStream(Teuchos::rcpFromRef(cout));
// since A is chosen symmetric, it is P^T = R
// check P^T * P = R * P
// note: the Epetra matrix-matrix multiplication using implicit transpose is buggy in parallel case
// (for multiplication of a square matrix with a rectangular matrix)
// however it seems to work for two rectangular matrices
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > RP = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*R1,false,*P1,false,out);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > PtP = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*P1,true,*P1,false,out);
RCP<Vector> x = VectorFactory::Build(RP->getDomainMap());
RCP<Vector> bRP = VectorFactory::Build(RP->getRangeMap());
RCP<Vector> bPtP = VectorFactory::Build(PtP->getRangeMap());
x->randomize();
RP->apply(*x,*bRP);
PtP->apply(*x,*bPtP);
TEST_EQUALITY(bRP->norm1() - bPtP->norm1() < 1e-12, true);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > RP2 = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*R2,false,*P2,false,out);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > PtP2 = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*P2,true,*P2,false,out);
x = VectorFactory::Build(RP2->getDomainMap());
bRP = VectorFactory::Build(RP2->getRangeMap());
bPtP = VectorFactory::Build(PtP2->getRangeMap());
x->randomize();
RP2->apply(*x,*bRP);
PtP2->apply(*x,*bPtP);
TEST_EQUALITY(bRP->norm1() - bPtP->norm1() < 1e-12, true);
//R1->describe(*fos,Teuchos::VERB_EXTREME);
}
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> > &inA,
Teuchos::ParameterList& inParamList,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node>>& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>>& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
typedef Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> crs_matrix_type;
typedef Tpetra::Experimental::BlockCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> block_crs_matrix_type;
bool hasParamList = inParamList.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = inParamList;
RCP<const crs_matrix_type> constCrsA;
RCP<crs_matrix_type> crsA;
#if defined(HAVE_MUELU_EXPERIMENTAL) and defined(HAVE_MUELU_AMGX)
std::string externalMG = "use external multigrid package";
if (hasParamList && paramList.isParameter(externalMG) && paramList.get<std::string>(externalMG) == "amgx"){
constCrsA = rcp_dynamic_cast<const crs_matrix_type>(inA);
TEUCHOS_TEST_FOR_EXCEPTION(constCrsA == Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: failed to dynamic cast to Tpetra::CrsMatrix, which is required to be able to use AmgX.");
return rcp(new AMGXOperator<SC,LO,GO,NO>(inA,inParamList));
}
#endif
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList,inA->getDomainMap()->getComm()));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A;
RCP<block_crs_matrix_type> bcrsA = rcp_dynamic_cast<block_crs_matrix_type>(inA);
crsA = rcp_dynamic_cast<crs_matrix_type>(inA);
if (crsA != Teuchos::null)
A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(crsA);
else if (bcrsA != Teuchos::null) {
RCP<Xpetra::CrsMatrix<SC,LO,GO,NO> > temp = rcp(new Xpetra::TpetraBlockCrsMatrix<SC,LO,GO,NO>(bcrsA));
TEUCHOS_TEST_FOR_EXCEPTION(temp==Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: cast from Tpetra::Experimental::BlockCrsMatrix to Xpetra::TpetraBlockCrsMatrix failed.");
A = rcp(new Xpetra::CrsMatrixWrap<SC,LO,GO,NO>(temp));
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "CreateTpetraPreconditioner: only Tpetra CrsMatrix and BlockCrsMatrix types are supported.");
}
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP; using Teuchos::rcp;
using Teuchos::TimeMonitor;
//using Galeri::Xpetra::CreateCartesianCoordinates;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// out->setOutputToRootOnly(-1);
// out->precision(12);
//FIXME we need a HAVE_MUELU_LONG_LONG_INT option
//#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
//#endif
//
// SET TEST PARAMETERS
//
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561 = 3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748 = 4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// Custom command line parameters
// - Debug
int optDebug = 0; clp.setOption("debug", &optDebug, "pause to attach debugger");
int optDump = 0; clp.setOption("dump", &optDump, "write matrix to file");
int optTimings = 0; clp.setOption("timings", &optTimings, "print timings to screen");
// - Levels
LO optMaxLevels = 10; clp.setOption("maxLevels", &optMaxLevels, "maximum number of levels allowed");
int optMaxCoarseSize = 50; clp.setOption("maxCoarseSize", &optMaxCoarseSize, "maximum #dofs in coarse operator"); //FIXME clp doesn't like long long int
// - Smoothed-Aggregation
Scalar optSaDamping = 4./3; clp.setOption("saDamping", &optSaDamping, "prolongator damping factor");
// - Aggregation
std::string optAggOrdering = "natural"; clp.setOption("aggOrdering", &optAggOrdering, "aggregation ordering strategy (natural, random, graph)");
int optMinPerAgg = 2; clp.setOption("minPerAgg", &optMinPerAgg, "minimum #DOFs per aggregate");
int optMaxNbrSel = 0; clp.setOption("maxNbrSel", &optMaxNbrSel, "maximum # of nbrs allowed to be in other aggregates");
// - R
int optExplicitR = 1; clp.setOption("explicitR", &optExplicitR, "restriction will be explicitly stored as transpose of prolongator");
// - Smoothers
std::string optSmooType = "sgs"; clp.setOption("smooType", &optSmooType, "smoother type ('l1-sgs', 'sgs 'or 'cheby')");
int optSweeps = 2; clp.setOption("sweeps", &optSweeps, "sweeps to be used in SGS (or Chebyshev degree)");
// - Repartitioning
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
int optRepartition = 1; clp.setOption("repartition", &optRepartition, "enable repartitioning (0=no repartitioning, 1=Zoltan RCB, 2=Isorropia+Zoltan PHG");
LO optMinRowsPerProc = 2000; clp.setOption("minRowsPerProc", &optMinRowsPerProc, "min #rows allowable per proc before repartitioning occurs");
double optNnzImbalance = 1.2; clp.setOption("nnzImbalance", &optNnzImbalance, "max allowable nonzero imbalance before repartitioning occurs");
#else
int optRepartition = 0;
#endif // HAVE_MPI && HAVE_MUELU_ZOLTAN
// - Solve
int optFixPoint = 1; clp.setOption("fixPoint", &optFixPoint, "apply multigrid as solver");
int optPrecond = 1; clp.setOption("precond", &optPrecond, "apply multigrid as preconditioner");
LO optIts = 10; clp.setOption("its", &optIts, "number of multigrid cycles");
double optTol = 1e-7; clp.setOption("tol", &optTol, "stopping tolerance for Krylov method");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
if (optDebug) {
Utils::PauseForDebugger();
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
std::transform(optSmooType.begin(), optSmooType.end(), optSmooType.begin(), ::tolower);
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
if (comm->getRank() == 0) {
std::cout << xpetraParameters << matrixParameters;
// TODO: print custom parameters // Or use paramList::print()!
}
//
// CREATE INITIAL MATRIX */
//
RCP<const Map> map;
RCP<Matrix> A;
RCP<MultiVector> coordinates;
{
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build"));
map = MapFactory::Build(lib, matrixParameters.GetNumGlobalElements(), 0, comm);
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
A = Pr->BuildMatrix();
if (matrixParameters.GetMatrixType() == "Laplace1D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
}
//
//
//
// dump matrix to file
if (optDump) {
std::string fileName = "Amat.mm";
Utils::Write(fileName, *A);
}
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullspace->norm1(norms);
if (comm->getRank() == 0)
std::cout << "||NS|| = " << norms[0] << std::endl;
RCP<MueLu::Hierarchy<SC, LO, GO, NO, LMO> > H;
//
//
// SETUP
//
//
{
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup"));
//
// Hierarchy
//
H = rcp(new Hierarchy());
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
H->SetMaxCoarseSize((GO) optMaxCoarseSize);
//
// Finest level
//
RCP<Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A", A);
Finest->Set("Nullspace", nullspace);
Finest->Set("Coordinates", coordinates); //FIXME: XCoordinates, YCoordinates, ..
//
// FactoryManager
//
FactoryManager M;
//
//
// Aggregation
//
{
RCP<UncoupledAggregationFactory> AggregationFact = rcp(new UncoupledAggregationFactory());
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min DOFs per aggregate : " << optMinPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << optMaxNbrSel << std::endl;
AggregationFact->SetMinNodesPerAggregate(optMinPerAgg); //TODO should increase if run anything othpermRFacter than 1D
AggregationFact->SetMaxNeighAlreadySelected(optMaxNbrSel);
std::transform(optAggOrdering.begin(), optAggOrdering.end(), optAggOrdering.begin(), ::tolower);
if (optAggOrdering == "natural") {
*out << "aggregate ordering : NATURAL" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::NATURAL);
} else if (optAggOrdering == "random") {
*out << "aggregate ordering : RANDOM" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::RANDOM);
} else if (optAggOrdering == "graph") {
*out << "aggregate ordering : GRAPH" << std::endl;
AggregationFact->SetOrdering(MueLu::AggOptions::GRAPH);
} else {
std::string msg = "main: bad aggregation option """ + optAggOrdering + """.";
throw(MueLu::Exceptions::RuntimeError(msg));
}
//AggregationFact->SetPhase3AggCreation(0.5);
M.SetFactory("Aggregates", AggregationFact);
*out << "=============================================================================" << std::endl;
}
//
// Transfer
//
{
//
// Non rebalanced factories
//
RCP<SaPFactory> PFact = rcp(new SaPFactory());
PFact->SetDampingFactor(optSaDamping);
RCP<Factory> RFact = rcp(new TransPFactory());
RCP<RAPFactory> AFact = rcp(new RAPFactory());
AFact->setVerbLevel(Teuchos::VERB_HIGH);
if (!optExplicitR) {
H->SetImplicitTranspose(true);
ParameterList Aclist = *(AFact->GetValidParameterList());
Aclist.set("implicit transpose", true);
AFact->SetParameterList(Aclist);
if (comm->getRank() == 0) std::cout << "\n\n* ***** USING IMPLICIT RESTRICTION OPERATOR ***** *\n" << std::endl;
}
//
// Repartitioning (if needed)
//
if (optRepartition == 0) {
// No repartitioning
// Configure FactoryManager
M.SetFactory("P", PFact);
M.SetFactory("R", RFact);
M.SetFactory("A", AFact);
} else {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN)
// Repartitioning
// The Factory Manager will be configured to return the rebalanced versions of P, R, A by default.
// Everytime we want to use the non-rebalanced versions, we need to explicitly define the generating factory.
RFact->SetFactory("P", PFact);
//
AFact->SetFactory("P", PFact);
AFact->SetFactory("R", RFact);
// Transfer coordinates
RCP<CoordinatesTransferFactory> TransferCoordinatesFact = rcp(new CoordinatesTransferFactory());
AFact->AddTransferFactory(TransferCoordinatesFact); // FIXME REMOVE
// Compute partition (creates "Partition" object)
if(optRepartition == 1) { // use plain Zoltan Interface
} else if (optRepartition == 2) { // use Isorropia + Zoltan interface
}
// Repartitioning (creates "Importer" from "Partition")
RCP<Factory> RepartitionFact = rcp(new RepartitionFactory());
{
Teuchos::ParameterList paramList;
paramList.set("minRowsPerProcessor", optMinRowsPerProc);
paramList.set("nonzeroImbalance", optNnzImbalance);
RepartitionFact->SetParameterList(paramList);
}
RepartitionFact->SetFactory("A", AFact);
if(optRepartition == 1) {
RCP<Factory> ZoltanFact = rcp(new ZoltanInterface());
ZoltanFact->SetFactory("A", AFact);
ZoltanFact->SetFactory("Coordinates", TransferCoordinatesFact);
RepartitionFact->SetFactory("Partition", ZoltanFact);
}
else if(optRepartition == 2) {
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ISORROPIA)
RCP<MueLu::IsorropiaInterface<LO, GO, NO, LMO> > isoInterface = rcp(new MueLu::IsorropiaInterface<LO, GO, NO, LMO>());
isoInterface->SetFactory("A", AFact);
// we don't need Coordinates here!
RepartitionFact->SetFactory("Partition", isoInterface);
#else
if (comm->getRank() == 0)
std::cout << "Please recompile Trilinos with Isorropia support enabled." << std::endl;
return EXIT_FAILURE;
#endif
}
// Reordering of the transfer operators
RCP<Factory> RebalancedPFact = rcp(new RebalanceTransferFactory());
RebalancedPFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Interpolation")));
RebalancedPFact->SetFactory("P", PFact);
RebalancedPFact->SetFactory("Coordinates", TransferCoordinatesFact);
RebalancedPFact->SetFactory("Nullspace", M.GetFactory("Ptent")); // TODO
RCP<Factory> RebalancedRFact = rcp(new RebalanceTransferFactory());
RebalancedRFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Restriction")));
RebalancedRFact->SetFactory("R", RFact);
// Compute Ac from rebalanced P and R
RCP<Factory> RebalancedAFact = rcp(new RebalanceAcFactory());
RebalancedAFact->SetFactory("A", AFact);
// Configure FactoryManager
M.SetFactory("A", RebalancedAFact);
M.SetFactory("P", RebalancedPFact);
M.SetFactory("R", RebalancedRFact);
M.SetFactory("Nullspace", RebalancedPFact);
M.SetFactory("Coordinates", RebalancedPFact);
M.SetFactory("Importer", RepartitionFact);
#else
TEUCHOS_TEST_FOR_EXCEPT(true);
#endif
} // optRepartition
} // Transfer
//
// Smoothers
//
{
std::string ifpackType;
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) optSweeps);
ifpackList.set("relaxation: damping factor", (SC) 1.0);
if (optSmooType == "sgs") {
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
}
else if (optSmooType == "l1-sgs") {
ifpackType = "RELAXATION";
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
ifpackList.set("relaxation: use l1", true);
} else if (optSmooType == "cheby") {
ifpackType = "CHEBYSHEV";
ifpackList.set("chebyshev: degree", (LO) optSweeps);
if (matrixParameters.GetMatrixType() == "Laplace1D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 3);
}
else if (matrixParameters.GetMatrixType() == "Laplace2D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 7);
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
ifpackList.set("chebyshev: ratio eigenvalue", (SC) 20);
}
// ifpackList.set("chebyshev: max eigenvalue", (double) -1.0);
// ifpackList.set("chebyshev: min eigenvalue", (double) 1.0);
}
RCP<SmootherPrototype> smootherPrototype = rcp(new TrilinosSmoother(ifpackType, ifpackList));
M.SetFactory("Smoother", rcp(new SmootherFactory(smootherPrototype)));
}
//
// Setup preconditioner
//
int startLevel = 0;
// std::cout << startLevel << " " << optMaxLevels << std::endl;
H->Setup(M, startLevel, optMaxLevels);
} // end of Setup TimeMonitor
/*{ // some debug output
// print out content of levels
std::cout << "FINAL CONTENT of multigrid levels" << std::endl;
for(LO l = 0; l < H->GetNumLevels(); l++) {
RCP<Level> coarseLevel = H->GetLevel(l);
coarseLevel->print(*out);
}
std::cout << "END FINAL CONTENT of multigrid levels" << std::endl;
} // end debug output*/
//
//
// SOLVE
//
//
// Define X, B
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
X->setSeed(846930886);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
B->norm2(norms);
B->scale(1.0/norms[0]);
//
// Use AMG directly as an iterative method
//
if (optFixPoint) {
X->putScalar( (SC) 0.0);
TimeMonitor tm(*TimeMonitor::getNewTimer("ScalingTest: 3 - Fixed Point Solve"));
H->IsPreconditioner(false);
H->Iterate(*B, *X, optIts);
} // optFixedPt
//
// Use AMG as a preconditioner in Belos
//
#ifdef HAVE_MUELU_BELOS
if (optPrecond) {
RCP<TimeMonitor> tm;
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Operator object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 100;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", optTol); // Relative convergence tolerance requested
//belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
{
TimeMonitor tm2(*TimeMonitor::getNewTimer("ScalingTest: 5bis - Belos Internal Solve"));
ret = solver->solve();
} // end of TimeMonitor
// Get the number of iterations for this solve.
if (comm->getRank() == 0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
// Compute actual residuals.
int numrhs = 1;
std::vector<double> actual_resids( numrhs ); //TODO: double?
std::vector<double> rhs_norm( numrhs );
RCP<MultiVector> resid = MultiVectorFactory::Build(map, numrhs);
typedef Belos::OperatorTraits<SC, MV, OP> OPT;
typedef Belos::MultiVecTraits<SC, MV> MVT;
OPT::Apply( *belosOp, *X, *resid );
MVT::MvAddMv( -1.0, *resid, 1.0, *B, *resid );
MVT::MvNorm( *resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
*out<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for ( int i = 0; i<numrhs; i++) {
double actRes = actual_resids[i]/rhs_norm[i];
*out<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
//if (actRes > tol) { badRes = true; }
}
} //try
catch(...) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged) {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
} else {
if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
}
tm = Teuchos::null;
} //if (optPrecond)
#endif // HAVE_MUELU_BELOS
//
// Timer final summaries
//
globalTimeMonitor = Teuchos::null; // stop this timer before summary
if (optTimings)
TimeMonitor::summarize();
//
return EXIT_SUCCESS;
}
TEUCHOS_UNIT_TEST(SaPFactory_kokkos, EpetraVsTpetra)
{
# include "MueLu_UseShortNames.hpp"
MueLu::VerboseObject::SetDefaultOStream(Teuchos::rcpFromRef(out));
out << "version: " << MueLu::Version() << std::endl;
out << "Compare results of Epetra and Tpetra" << std::endl;
out << "for 3 level AMG solver using smoothed aggregation with" << std::endl;
out << "one SGS sweep on each multigrid level as pre- and postsmoother" << std::endl;
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
Array<STS::magnitudeType> results(2);
// run test only on 1 proc
if(comm->getSize() == 1) {
Xpetra::UnderlyingLib lib = Xpetra::UseEpetra;
// run Epetra and Tpetra test
for (int run = 0; run < 2; run++) { //TODO: create a subfunction instead or Tuple of UnderlyingLib
if (run == 0) lib = Xpetra::UseEpetra;
else lib = Xpetra::UseTpetra;
// generate problem
LO maxLevels = 3;
LO its = 10;
GO nEle = 63;
const RCP<const Map> map = MapFactory::Build(lib, nEle, 0, comm);
Teuchos::ParameterList matrixParameters;
matrixParameters.set("nx", nEle);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>("Laplace1D", map, matrixParameters);
RCP<Matrix> Op = Pr->BuildMatrix();
// build nullspace
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar(one);
Array<STS::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
out << "||NS|| = " << norms[0] << std::endl;
// fill hierarchy
RCP<Hierarchy> H = rcp( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
RCP<Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op); // set fine level matrix
Finest->Set("Nullspace",nullSpace); // set null space information for finest level
// define transfer operators
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetMinNodesPerAggregate(3);
CoupledAggFact->SetMaxNeighAlreadySelected(0);
CoupledAggFact->SetOrdering("natural");
CoupledAggFact->SetPhase3AggCreation(0.5);
RCP<TentativePFactory> Ptentfact = rcp(new TentativePFactory());
RCP<SaPFactory> Pfact = rcp( new SaPFactory());
RCP<Factory> Rfact = rcp( new TransPFactory() );
RCP<RAPFactory> Acfact = rcp( new RAPFactory() );
H->SetMaxCoarseSize(1);
// setup smoothers
Teuchos::ParameterList smootherParamList;
smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
smootherParamList.set("relaxation: sweeps", (LO) 1);
smootherParamList.set("relaxation: damping factor", (SC) 1.0);
RCP<SmootherPrototype> smooProto = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );
RCP<SmootherFactory> SmooFact = rcp( new SmootherFactory(smooProto) );
Acfact->setVerbLevel(Teuchos::VERB_HIGH);
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(smooProto, Teuchos::null));
FactoryManager M;
M.SetFactory("P", Pfact);
M.SetFactory("R", Rfact);
M.SetFactory("A", Acfact);
M.SetFactory("Ptent", Ptentfact);
M.SetFactory("Aggregates", CoupledAggFact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", coarseSolveFact);
H->Setup(M, 0, maxLevels);
// test some basic multigrid data
RCP<Level> coarseLevel = H->GetLevel(1);
TEST_EQUALITY(coarseLevel->IsRequested("A",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("P",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PreSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PostSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("R",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("A",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("P",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PreSmoother",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PostSmoother",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("R",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->GetKeepFlag("A",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("P",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PreSmoother",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PostSmoother",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("R",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->IsRequested("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PreSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PostSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("A",Acfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("PreSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("PostSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("A",Acfact.get()), false);
TEST_EQUALITY(coarseLevel->GetKeepFlag("P",Pfact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("P",Ptentfact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PreSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PostSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("R",Rfact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("A",Acfact.get()), 0);
RCP<Matrix> P1 = coarseLevel->Get< RCP<Matrix> >("P");
RCP<Matrix> R1 = coarseLevel->Get< RCP<Matrix> >("R");
TEST_EQUALITY(P1->getGlobalNumRows(), 63);
TEST_EQUALITY(P1->getGlobalNumCols(), 21);
TEST_EQUALITY(R1->getGlobalNumRows(), 21);
TEST_EQUALITY(R1->getGlobalNumCols(), 63);
RCP<Level> coarseLevel2 = H->GetLevel(2);
TEST_EQUALITY(coarseLevel2->IsRequested("A",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("P",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("R",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("PreSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("PostSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("A",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("P",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PreSmoother",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PostSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("R",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("A",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("P",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PreSmoother",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PostSmoother",MueLu::NoFactory::get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("R",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->IsRequested("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("PreSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("PostSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("P",Pfact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("P",Ptentfact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PreSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PostSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("R",Rfact.get()), 0);
RCP<Matrix> P2 = coarseLevel2->Get< RCP<Matrix> >("P");
RCP<Matrix> R2 = coarseLevel2->Get< RCP<Matrix> >("R");
TEST_EQUALITY(P2->getGlobalNumRows(), 21);
TEST_EQUALITY(P2->getGlobalNumCols(), 7);
TEST_EQUALITY(R2->getGlobalNumRows(), 7);
TEST_EQUALITY(R2->getGlobalNumCols(), 21);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO> > PtentTPtent = Xpetra::MatrixMatrix<Scalar,LO,GO>::Multiply(*P1,true,*P1,false,out);
TEST_EQUALITY(PtentTPtent->getGlobalMaxNumRowEntries()-3<1e-12, true);
TEST_EQUALITY(P1->getGlobalMaxNumRowEntries()-2<1e-12, true);
TEST_EQUALITY(P2->getGlobalMaxNumRowEntries()-2<1e-12, true);
// Define RHS
RCP<MultiVector> X = MultiVectorFactory::Build(map,1);
RCP<MultiVector> RHS = MultiVectorFactory::Build(map,1);
X->putScalar(1.0);
X->norm2(norms);
if (comm->getRank() == 0)
out << "||X_true|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
Op->apply(*X,*RHS,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
// Use AMG directly as an iterative method
{
X->putScalar( (SC) 0.0);
H->Iterate(*RHS,*X,its);
X->norm2(norms);
if (comm->getRank() == 0)
out << "||X_" << std::setprecision(2) << its << "|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
results[run] = norms[0];
}
}
TEST_EQUALITY(results[0] - results[1] < 1e-10, true); // check results of EPETRA vs TPETRA
} // comm->getSize == 1
} //SaPFactory_EpetraVsTpetra
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcpFromRef;
using namespace MueLuTests;
#ifdef __GNUC__
#warning Navier2DBlocked_test based tests are disabled on 12/11/2013 due to some thrown exception
#endif
return EXIT_SUCCESS;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
//
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// Timing
Teuchos::Time myTime("global");
Teuchos::TimeMonitor MM(myTime);
// read in some command line parameters
Teuchos::CommandLineProcessor clp(false);
int rebalanceBlock0 = 1; clp.setOption("rebalanceBlock0", &rebalanceBlock0, "rebalance block 0 (1=yes, else=no)");
int rebalanceBlock1 = 1; clp.setOption("rebalanceBlock1", &rebalanceBlock1, "rebalance block 1 (1=yes, else=no)");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
#if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN) && defined(HAVE_MUELU_ISORROPIA)
#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
// custom parameters
LocalOrdinal maxLevels = 3;
GlobalOrdinal maxCoarseSize=1; //FIXME clp doesn't like long long int
int globalNumDofs = 8898; // used for the maps
int nDofsPerNode = 3; // used for generating the fine level null-space
// build strided maps
// striding information: 2 velocity dofs and 1 pressure dof = 3 dofs per node
std::vector<size_t> stridingInfo;
stridingInfo.push_back(2);
stridingInfo.push_back(1);
/////////////////////////////////////// build strided maps
// build strided maps:
// xstridedfullmap: full map (velocity and pressure dof gids), continous
// xstridedvelmap: only velocity dof gid maps (i.e. 0,1,3,4,6,7...)
// xstridedpremap: only pressure dof gid maps (i.e. 2,5,8,...)
Xpetra::UnderlyingLib lib = Xpetra::UseEpetra;
RCP<StridedMap> xstridedfullmap = StridedMapFactory::Build(lib,globalNumDofs,0,stridingInfo,comm,-1);
RCP<StridedMap> xstridedvelmap = StridedMapFactory::Build(xstridedfullmap,0);
RCP<StridedMap> xstridedpremap = StridedMapFactory::Build(xstridedfullmap,1);
/////////////////////////////////////// transform Xpetra::Map objects to Epetra
// this is needed for AztecOO
const RCP<const Epetra_Map> fullmap = rcpFromRef(Xpetra::toEpetra(*xstridedfullmap));
RCP<const Epetra_Map> velmap = rcpFromRef(Xpetra::toEpetra(*xstridedvelmap));
RCP<const Epetra_Map> premap = rcpFromRef(Xpetra::toEpetra(*xstridedpremap));
/////////////////////////////////////// import problem matrix and RHS from files (-> Epetra)
// read in problem
Epetra_CrsMatrix * ptrA = 0;
Epetra_Vector * ptrf = 0;
Epetra_MultiVector* ptrNS = 0;
*out << "Reading matrix market file" << std::endl;
EpetraExt::MatrixMarketFileToCrsMatrix("A5932_re1000.txt",*fullmap,*fullmap,*fullmap,ptrA);
EpetraExt::MatrixMarketFileToVector("b5932_re1000.txt",*fullmap,ptrf);
//EpetraExt::MatrixMarketFileToCrsMatrix("/home/tobias/promotion/trilinos/fc17-dyn/packages/muelu/test/navierstokes/A5932_re1000.txt",*fullmap,*fullmap,*fullmap,ptrA);
//EpetraExt::MatrixMarketFileToVector("/home/tobias/promotion/trilinos/fc17-dyn/packages/muelu/test/navierstokes/b5932_re1000.txt",*fullmap,ptrf);
RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
RCP<Epetra_Vector> epv = Teuchos::rcp(ptrf);
RCP<Epetra_MultiVector> epNS = Teuchos::rcp(ptrNS);
/////////////////////////////////////// split system into 2x2 block system
*out << "Split matrix into 2x2 block matrix" << std::endl;
// split fullA into A11,..., A22
Teuchos::RCP<Epetra_CrsMatrix> A11;
Teuchos::RCP<Epetra_CrsMatrix> A12;
Teuchos::RCP<Epetra_CrsMatrix> A21;
Teuchos::RCP<Epetra_CrsMatrix> A22;
if(SplitMatrix2x2(epA,*velmap,*premap,A11,A12,A21,A22)==false)
*out << "Problem with splitting matrix"<< std::endl;
/////////////////////////////////////// transform Epetra objects to Xpetra (needed for MueLu)
// build Xpetra objects from Epetra_CrsMatrix objects
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA11 = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(A11));
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA12 = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(A12));
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA21 = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(A21));
Teuchos::RCP<Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > xA22 = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(A22));
/////////////////////////////////////// generate MapExtractor object
std::vector<Teuchos::RCP<const Xpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > > xmaps;
xmaps.push_back(xstridedvelmap);
xmaps.push_back(xstridedpremap);
Teuchos::RCP<const Xpetra::MapExtractor<Scalar,LocalOrdinal,GlobalOrdinal,Node> > map_extractor = Xpetra::MapExtractorFactory<Scalar,LocalOrdinal,GlobalOrdinal>::Build(xstridedfullmap,xmaps);
/////////////////////////////////////// build blocked transfer operator
// using the map extractor
Teuchos::RCP<Xpetra::BlockedCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > bOp = Teuchos::rcp(new Xpetra::BlockedCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal>(map_extractor,map_extractor,10));
bOp->setMatrix(0,0,xA11);
bOp->setMatrix(0,1,xA12);
bOp->setMatrix(1,0,xA21);
bOp->setMatrix(1,1,xA22);
bOp->fillComplete();
//////////////////////////////////////////////////// create Hierarchy
RCP<Hierarchy> H = rcp ( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
//H->setDefaultVerbLevel(Teuchos::VERB_NONE);
H->SetMaxCoarseSize(maxCoarseSize);
//////////////////////////////////////////////////////// finest Level
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Teuchos::rcp_dynamic_cast<Matrix>(bOp));
////////////////////////////////////////// prepare null space for A11
RCP<MultiVector> nullspace11 = MultiVectorFactory::Build(xstridedvelmap, 2); // this is a 2D standard null space
for (int i=0; i<nDofsPerNode-1; ++i) {
Teuchos::ArrayRCP<Scalar> nsValues = nullspace11->getDataNonConst(i);
int numBlocks = nsValues.size() / (nDofsPerNode - 1);
for (int j=0; j< numBlocks; ++j) {
nsValues[j*(nDofsPerNode - 1) + i] = 1.0;
}
}
Finest->Set("Nullspace1",nullspace11);
////////////////////////////////////////// prepare null space for A22
RCP<MultiVector> nullspace22 = MultiVectorFactory::Build(xstridedpremap, 1); // this is a 2D standard null space
Teuchos::ArrayRCP<Scalar> nsValues22 = nullspace22->getDataNonConst(0);
for (int j=0; j< nsValues22.size(); ++j) {
nsValues22[j] = 1.0;
}
Finest->Set("Nullspace2",nullspace22);
/////////////////////////////////////////// define rebalanced block AC factory
// This is the main factory for "A" and defines the input for
// - the SubBlockAFactory objects
// - the rebalanced block Ac factory
RCP<RebalanceBlockAcFactory> RebalancedAcFact = rcp(new RebalanceBlockAcFactory());
/////////////////////////////////////////// define non-rebalanced blocked transfer ops
RCP<BlockedPFactory> PFact = rcp(new BlockedPFactory()); // use row map index base from bOp
RCP<GenericRFactory> RFact = rcp(new GenericRFactory());
RFact->SetFactory("P", PFact);
// non-rebalanced block coarse matrix factory
// output is non-rebalanced coarse block matrix Ac
// used as input for rebalanced block coarse factory RebalancedAcFact
RCP<Factory> AcFact = rcp(new BlockedRAPFactory());
AcFact->SetFactory("A", MueLu::NoFactory::getRCP());
AcFact->SetFactory("P", PFact); // use non-rebalanced block prolongator as input
AcFact->SetFactory("R", RFact); // use non-rebalanced block restrictor as input
// define matrix sub-blocks of possibly rebalanced block matrix A
// These are used as input for
// - the sub blocks of the transfer operators
RCP<SubBlockAFactory> A11Fact = Teuchos::rcp(new SubBlockAFactory());
A11Fact->SetFactory("A",MueLu::NoFactory::getRCP());
A11Fact->SetParameter("block row",Teuchos::ParameterEntry(0));
A11Fact->SetParameter("block col",Teuchos::ParameterEntry(0));
RCP<SubBlockAFactory> A22Fact = Teuchos::rcp(new SubBlockAFactory());
A22Fact->SetFactory("A",MueLu::NoFactory::getRCP());
A22Fact->SetParameter("block row",Teuchos::ParameterEntry(1));
A22Fact->SetParameter("block col",Teuchos::ParameterEntry(1));
/////////////////////////////////////////// define rebalancing factories
// define sub blocks of the coarse non-rebalanced block matrix Ac
// input is the block operator generated by AcFact
RCP<SubBlockAFactory> rebA11Fact = Teuchos::rcp(new SubBlockAFactory());
rebA11Fact->SetFactory("A",AcFact);
rebA11Fact->SetParameter("block row",Teuchos::ParameterEntry(0));
rebA11Fact->SetParameter("block col",Teuchos::ParameterEntry(0));
RCP<SubBlockAFactory> rebA22Fact = Teuchos::rcp(new SubBlockAFactory());
rebA22Fact->SetFactory("A",AcFact);
rebA22Fact->SetParameter("block row",Teuchos::ParameterEntry(1));
rebA22Fact->SetParameter("block col",Teuchos::ParameterEntry(1));
// define rebalancing factory for coarse block matrix A(1,1)
RCP<AmalgamationFactory> rebAmalgFact11 = rcp(new AmalgamationFactory());
rebAmalgFact11->SetFactory("A", rebA11Fact);
rebAmalgFact11->setDefaultVerbLevel(Teuchos::VERB_EXTREME);
RCP<MueLu::IsorropiaInterface<LO, GO, NO, LMO> > isoInterface1 = rcp(new MueLu::IsorropiaInterface<LO, GO, NO, LMO>());
isoInterface1->SetFactory("A", rebA11Fact);
isoInterface1->SetFactory("UnAmalgamationInfo", rebAmalgFact11);
RCP<MueLu::RepartitionInterface<LO, GO, NO, LMO> > repInterface1 = rcp(new MueLu::RepartitionInterface<LO, GO, NO, LMO>());
repInterface1->SetFactory("A", rebA11Fact);
repInterface1->SetFactory("AmalgamatedPartition", isoInterface1);
repInterface1->SetFactory("UnAmalgamationInfo", rebAmalgFact11);
// Repartitioning (creates "Importer" from "Partition")
RCP<Factory> RepartitionFact = rcp(new RepartitionFactory());
{
Teuchos::ParameterList paramList;
paramList.set("minRowsPerProcessor", 200);
paramList.set("nonzeroImbalance", 1.3);
if(rebalanceBlock0 == 1)
paramList.set("startLevel",1);
else
paramList.set("startLevel",10); // supress rebalancing
RepartitionFact->SetParameterList(paramList);
}
RepartitionFact->SetFactory("A", rebA11Fact);
RepartitionFact->SetFactory("Partition", repInterface1);
// define rebalancing factory for coarse block matrix A(1,1)
RCP<AmalgamationFactory> rebAmalgFact22 = rcp(new AmalgamationFactory());
rebAmalgFact22->SetFactory("A", rebA22Fact);
rebAmalgFact22->setDefaultVerbLevel(Teuchos::VERB_EXTREME);
RCP<MueLu::IsorropiaInterface<LO, GO, NO, LMO> > isoInterface2 = rcp(new MueLu::IsorropiaInterface<LO, GO, NO, LMO>());
isoInterface2->SetFactory("A", rebA22Fact);
isoInterface2->SetFactory("UnAmalgamationInfo", rebAmalgFact22);
RCP<MueLu::RepartitionInterface<LO, GO, NO, LMO> > repInterface2 = rcp(new MueLu::RepartitionInterface<LO, GO, NO, LMO>());
repInterface2->SetFactory("A", rebA22Fact);
repInterface2->SetFactory("AmalgamatedPartition", isoInterface2);
repInterface2->SetFactory("UnAmalgamationInfo", rebAmalgFact22);
// second repartition factory
RCP<Factory> RepartitionFact2 = rcp(new RepartitionFactory());
{
Teuchos::ParameterList paramList;
paramList.set("minRowsPerProcessor", 100);
paramList.set("nonzeroImbalance", 1.2);
if(rebalanceBlock1 == 1)
paramList.set("startLevel",1);
else
paramList.set("startLevel",10); // supress rebalancing
RepartitionFact2->SetParameterList(paramList);
}
RepartitionFact2->SetFactory("A", rebA22Fact);
RepartitionFact2->SetFactory("Partition", repInterface2); // this is not valid
////////////////////////////////////////// build non-rebalanced matrix blocks
// build factories for transfer operator P(1,1) and R(1,1)
RCP<AmalgamationFactory> amalgFact11 = rcp(new AmalgamationFactory());
amalgFact11->SetFactory("A", A11Fact);
amalgFact11->setDefaultVerbLevel(Teuchos::VERB_EXTREME);
RCP<CoalesceDropFactory> dropFact11 = rcp(new CoalesceDropFactory());
dropFact11->SetFactory("A", A11Fact);
dropFact11->SetFactory("UnAmalgamationInfo", amalgFact11);
dropFact11->setDefaultVerbLevel(Teuchos::VERB_EXTREME);
RCP<UncoupledAggregationFactory> UncoupledAggFact11 = rcp(new UncoupledAggregationFactory());
UncoupledAggFact11->SetFactory("Graph", dropFact11);
UncoupledAggFact11->SetMinNodesPerAggregate(9);
UncoupledAggFact11->SetMaxNeighAlreadySelected(2);
UncoupledAggFact11->SetOrdering(MueLu::AggOptions::NATURAL);
RCP<CoarseMapFactory> coarseMapFact11 = Teuchos::rcp(new CoarseMapFactory());
coarseMapFact11->setStridingData(stridingInfo);
coarseMapFact11->setStridedBlockId(0);
RCP<TentativePFactory> P11Fact = rcp(new TentativePFactory());
RCP<TransPFactory> R11Fact = rcp(new TransPFactory());
Teuchos::RCP<NullspaceFactory> nspFact11 = Teuchos::rcp(new NullspaceFactory("Nullspace1"));
nspFact11->SetFactory("Nullspace1",P11Fact); // pick "Nullspace1" from Finest level
//////////////////////////////// define factory manager for (1,1) block
RCP<FactoryManager> M11 = rcp(new FactoryManager());
M11->SetFactory("A", A11Fact); // rebalanced fine-level block operator
M11->SetFactory("P", P11Fact); // non-rebalanced transfer operator block P(1,1)
M11->SetFactory("R", R11Fact); // non-rebalanced transfer operator block R(1,1)
M11->SetFactory("Aggregates", UncoupledAggFact11);
M11->SetFactory("Graph", dropFact11);
M11->SetFactory("DofsPerNode", dropFact11);
M11->SetFactory("UnAmalgamationInfo", amalgFact11);
M11->SetFactory("Nullspace", nspFact11); // TODO check me?
M11->SetFactory("CoarseMap", coarseMapFact11);
M11->SetIgnoreUserData(true); // always use data from factories defined in factory manager
////////////////////////////////////////// build non-rebalanced matrix blocks
// build factories for transfer operator P(2,2) and R(2,2)
RCP<AmalgamationFactory> amalgFact22 = rcp(new AmalgamationFactory());
RCP<TentativePFactory> P22Fact = rcp(new TentativePFactory());
RCP<TransPFactory> R22Fact = rcp(new TransPFactory());
/*XXX*/
RCP<CoalesceDropFactory> dropFact22 = rcp(new CoalesceDropFactory());
dropFact22->SetFactory("A", A22Fact);
dropFact22->SetFactory("UnAmalgamationInfo", amalgFact22);
dropFact22->setDefaultVerbLevel(Teuchos::VERB_EXTREME);
/*XXX*/
RCP<UncoupledAggregationFactory> UncoupledAggFact22 = rcp(new UncoupledAggregationFactory());
UncoupledAggFact22->SetFactory("Graph", dropFact22);
UncoupledAggFact22->SetMinNodesPerAggregate(6);
UncoupledAggFact22->SetMaxNeighAlreadySelected(2);
UncoupledAggFact22->SetOrdering(MueLu::AggOptions::NATURAL);
// connect null space and tentative PFactory
Teuchos::RCP<NullspaceFactory> nspFact22 = Teuchos::rcp(new NullspaceFactory("Nullspace2"));
nspFact22->SetFactory("Nullspace2", P22Fact); // define null space generated by P22Fact as null space for coarse level (non-rebalanced)
RCP<CoarseMapFactory> coarseMapFact22 = Teuchos::rcp(new CoarseMapFactory());
coarseMapFact22->setStridingData(stridingInfo);
coarseMapFact22->setStridedBlockId(1);
//////////////////////////////// define factory manager for (2,2) block
RCP<FactoryManager> M22 = rcp(new FactoryManager());
M22->SetFactory("A", A22Fact); // rebalanced fine-level block operator
M22->SetFactory("P", P22Fact); // non-rebalanced transfer operator P(2,2)
M22->SetFactory("R", R22Fact); // non-rebalanced transfer operator R(2,2)
M22->SetFactory("Aggregates", UncoupledAggFact22 /* UncoupledAggFact11 *//*XXX*/); // aggregates from block (1,1)
M22->SetFactory("Graph", dropFact22);
M22->SetFactory("DofsPerNode", dropFact22);
M22->SetFactory("Nullspace", nspFact22); // TODO check me
M22->SetFactory("UnAmalgamationInfo", amalgFact22);
M22->SetFactory("Ptent", P22Fact);
M22->SetFactory("CoarseMap", coarseMapFact22);
M22->SetIgnoreUserData(true);
/////////////////////////////////////////// define rebalanced blocked transfer ops
//////////////////////////////// define factory manager for (1,1) block
RCP<FactoryManager> rebM11 = rcp(new FactoryManager());
rebM11->SetFactory("A", AcFact ); // important: must be a 2x2 block A Factory
//rebM11->SetFactory("P", PFact); // use non-rebalanced block P operator as input
//rebM11->SetFactory("R", RFact); // use non-rebalanced block R operator as input
rebM11->SetFactory("Importer", RepartitionFact);
rebM11->SetFactory("Nullspace", nspFact11);
//rebM11->SetIgnoreUserData(true);
RCP<FactoryManager> rebM22 = rcp(new FactoryManager());
rebM22->SetFactory("A", AcFact ); // important: must be a 2x2 block A Factory
rebM22->SetFactory("Importer", RepartitionFact2); // use dummy repartitioning factory
rebM22->SetFactory("Nullspace", nspFact22);
// Reordering of the transfer operators
RCP<RebalanceBlockInterpolationFactory> RebalancedBlockPFact = rcp(new RebalanceBlockInterpolationFactory());
RebalancedBlockPFact->SetFactory("P", PFact); // use non-rebalanced block P operator as input
RebalancedBlockPFact->AddFactoryManager(rebM11);
RebalancedBlockPFact->AddFactoryManager(rebM22);
RCP<RebalanceBlockRestrictionFactory> RebalancedBlockRFact = rcp(new RebalanceBlockRestrictionFactory());
//RebalancedBlockRFact->SetParameter("type", Teuchos::ParameterEntry(std::string("Restriction")));
RebalancedBlockRFact->SetFactory("R", RFact); // non-rebalanced block P operator
RebalancedBlockRFact->AddFactoryManager(rebM11);
RebalancedBlockRFact->AddFactoryManager(rebM22);
///////////////////////////////////////// initialize non-rebalanced block transfer operators
// output are the non-rebalanced block transfer operators used as input in AcFact to build
// the non-rebalanced coarse level block matrix Ac
PFact->AddFactoryManager(M11); // use non-rebalanced information from sub block factory manager M11
PFact->AddFactoryManager(M22); // use non-rebalanced information from sub block factory manager M22
///////////////////////////////////////// initialize rebalanced coarse block AC factory
RebalancedAcFact->SetFactory("A", AcFact); // use non-rebalanced block operator as input
RebalancedAcFact->AddFactoryManager(rebM11);
RebalancedAcFact->AddFactoryManager(rebM22);
//////////////////////////////////////////////////////////////////////
// Smoothers
//Another factory manager for braes sarazin smoother
//Schur Complement Factory, using the factory to generate AcFact
SC omega = 1.3;
RCP<SchurComplementFactory> SFact = Teuchos::rcp(new SchurComplementFactory());
SFact->SetParameter("omega", Teuchos::ParameterEntry(omega));
SFact->SetFactory("A", MueLu::NoFactory::getRCP()); // this finally be the rebalanced block operator!
//Smoother Factory, using SFact as a factory for A
std::string ifpackSCType;
Teuchos::ParameterList ifpackSCList;
ifpackSCList.set("relaxation: sweeps", (LocalOrdinal) 3);
ifpackSCList.set("relaxation: damping factor", (Scalar) 1.0);
ifpackSCType = "RELAXATION";
ifpackSCList.set("relaxation: type", "Gauss-Seidel");
RCP<SmootherPrototype> smoProtoSC = rcp( new TrilinosSmoother(ifpackSCType, ifpackSCList, 0) );
smoProtoSC->SetFactory("A", SFact);
RCP<SmootherFactory> SmooSCFact = rcp( new SmootherFactory(smoProtoSC) );
RCP<BraessSarazinSmoother> smootherPrototype = rcp( new BraessSarazinSmoother() );
smootherPrototype->SetParameter("Sweeps", Teuchos::ParameterEntry(3));
smootherPrototype->SetParameter("Damping factor", Teuchos::ParameterEntry(omega));
smootherPrototype->SetFactory("A",MueLu::NoFactory::getRCP());
RCP<SmootherFactory> smootherFact = rcp( new SmootherFactory(smootherPrototype) );
RCP<BraessSarazinSmoother> coarseSolverPrototype = rcp( new BraessSarazinSmoother() );
coarseSolverPrototype->SetParameter("Sweeps", Teuchos::ParameterEntry(3));
coarseSolverPrototype->SetParameter("Damping factor", Teuchos::ParameterEntry(omega));
coarseSolverPrototype->SetFactory("A",MueLu::NoFactory::getRCP());
RCP<SmootherFactory> coarseSolverFact = rcp( new SmootherFactory(coarseSolverPrototype, Teuchos::null) );
RCP<FactoryManager> MB = rcp(new FactoryManager());
MB->SetFactory("A", SFact);
MB->SetFactory("Smoother", SmooSCFact);
MB->SetIgnoreUserData(true); // always use data from factories defined in factory manager
smootherPrototype->AddFactoryManager(MB,0);
coarseSolverPrototype->AddFactoryManager(MB,0);
////////////////////////////////////////// define main factory manager
FactoryManager M;
M.SetFactory("A", RebalancedAcFact); // rebalance block AC Factory using importer
M.SetFactory("P", RebalancedBlockPFact); // rebalance prolongator using non-balanced Ac
M.SetFactory("R", RebalancedBlockRFact); // rebalance restrictor and null space using non-balanced Ac
M.SetFactory("Smoother", smootherFact);
M.SetFactory("PreSmoother", smootherFact);
M.SetFactory("PostSmoother", smootherFact);
M.SetFactory("CoarseSolver", coarseSolverFact);
H->Setup(M,0,maxLevels);
/**out << std::endl;
*out << "print content of multigrid levels:" << std::endl;
Finest->print(*out);
RCP<Level> coarseLevel = H->GetLevel(1);
coarseLevel->print(*out);
RCP<Level> coarseLevel2 = H->GetLevel(2);
coarseLevel2->print(*out);*/
RCP<MultiVector> xLsg = MultiVectorFactory::Build(xstridedfullmap,1);
// Use AMG directly as an iterative method
#if 0
{
xLsg->putScalar( (SC) 0.0);
// Epetra_Vector -> Xpetra::Vector
RCP<Vector> xRhs = Teuchos::rcp(new Xpetra::EpetraVector(epv));
// calculate initial (absolute) residual
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
xRhs->norm2(norms);
*out << "||x_0|| = " << norms[0] << std::endl;
// apply ten multigrid iterations
H->Iterate(*xRhs,100,*xLsg);
// calculate and print residual
RCP<MultiVector> xTmp = MultiVectorFactory::Build(xstridedfullmap,1);
bOp->apply(*xLsg,*xTmp,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
xRhs->update((SC)-1.0,*xTmp,(SC)1.0);
xRhs->norm2(norms);
*out << "||x|| = " << norms[0] << std::endl;
}
#endif
//
// Solve Ax = b using AMG as a preconditioner in AztecOO
//
{
RCP<Epetra_Vector> X = rcp(new Epetra_Vector(epv->Map()));
X->PutScalar(0.0);
Epetra_LinearProblem epetraProblem(epA.get(), X.get(), epv.get());
AztecOO aztecSolver(epetraProblem);
aztecSolver.SetAztecOption(AZ_solver, AZ_gmres);
MueLu::EpetraOperator aztecPrec(H);
aztecSolver.SetPrecOperator(&aztecPrec);
int maxIts = 50;
double tol = 1e-8;
aztecSolver.Iterate(maxIts, tol);
}
#endif // end ifndef HAVE_LONG_LONG_INT
#endif // #if defined(HAVE_MPI) && defined(HAVE_MUELU_ZOLTAN) && defined(HAVE_MUELU_ISORROPIA)
return EXIT_SUCCESS;
}
Teuchos::RCP<MueLu::Hierarchy<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateXpetraPreconditioner(Teuchos::RCP<Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > op,
const Teuchos::ParameterList& inParamList,
Teuchos::RCP<Xpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> > coords = Teuchos::null,
Teuchos::RCP<Xpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> > nullspace = Teuchos::null) {
typedef MueLu::HierarchyManager<Scalar,LocalOrdinal,GlobalOrdinal,Node> HierarchyManager;
typedef MueLu::HierarchyUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node> HierarchyUtils;
typedef MueLu::Hierarchy<Scalar,LocalOrdinal,GlobalOrdinal,Node> Hierarchy;
typedef MueLu::MLParameterListInterpreter<Scalar,LocalOrdinal,GlobalOrdinal,Node> MLParameterListInterpreter;
typedef MueLu::ParameterListInterpreter<Scalar,LocalOrdinal,GlobalOrdinal,Node> ParameterListInterpreter;
typedef Xpetra::MultiVectorFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node> MultiVectorFactory;
std::string timerName = "MueLu setup time";
RCP<Teuchos::Time> tm = Teuchos::TimeMonitor::getNewTimer(timerName);
tm->start();
bool hasParamList = inParamList.numParams();
RCP<HierarchyManager> mueLuFactory;
// Rip off non-serializable data before validation
Teuchos::ParameterList nonSerialList,paramList;
MueLu::ExtractNonSerializableData(inParamList, paramList, nonSerialList);
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter(paramList,op->getDomainMap()->getComm()));
}
// Create Hierarchy
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(op->getDomainMap()->lib());
// Stick the non-serializible data on the hierarchy.
HierarchyUtils::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
// Set fine level operator
H->GetLevel(0)->Set("A", op);
// Set coordinates if available
if (coords != Teuchos::null) {
H->GetLevel(0)->Set("Coordinates", coords);
}
// Wrap nullspace if available, otherwise use constants
if (nullspace == Teuchos::null) {
int nPDE = MueLu::MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = MultiVectorFactory::Build(op->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<Scalar>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<Scalar> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GlobalOrdinal GID = op->getDomainMap()->getGlobalElement(j) - op->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<Scalar>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
mueLuFactory->SetupHierarchy(*H);
tm->stop();
tm->incrementNumCalls();
if (H->GetVerbLevel() & Statistics0) {
const bool alwaysWriteLocal = true;
const bool writeGlobalStats = true;
const bool writeZeroTimers = false;
const bool ignoreZeroTimers = true;
const std::string filter = timerName;
Teuchos::TimeMonitor::summarize(op->getRowMap()->getComm().ptr(), std::cout, alwaysWriteLocal, writeGlobalStats,
writeZeroTimers, Teuchos::Union, filter, ignoreZeroTimers);
}
tm->reset();
return H;
}
// --------------------------------------------------------------------------------------
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
bool success = false;
bool verbose = true;
try {
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& out = *fancy;
typedef Teuchos::ScalarTraits<SC> STS;
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100, nz = 100;
Galeri::Xpetra::Parameters<GO> galeriParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
ParameterList galeriList = galeriParameters.GetParameterList();
out << thickSeparator << std::endl << xpetraParameters << galeriParameters;
// =========================================================================
// Problem construction
// =========================================================================
RCP<const Map> map;
RCP<Matrix> A,P,R, Ac;
RCP<MultiVector> nullspace;
std::string matrixType = galeriParameters.GetMatrixType();
MueLuExamples::generate_user_matrix_and_nullspace(matrixType,lib,galeriList,comm,A,nullspace);
map=A->getRowMap();
// =========================================================================
// Setups and solves
// =========================================================================
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
B->setSeed(846930886);
B->randomize();
RCP<TimeMonitor> tm;
#ifdef HAVE_MUELU_BELOS
// Belos Options
RCP<Teuchos::ParameterList> SList = rcp(new Teuchos::ParameterList );
SList->set("Verbosity",Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
SList->set("Output Frequency",10);
SList->set("Output Style",Belos::Brief);
SList->set("Maximum Iterations",200);
SList->set("Convergence Tolerance",1e-10);
#endif
// =========================================================================
// Solve #1 (standard MueLu)
// =========================================================================
out << thickSeparator << std::endl;
out << prefSeparator << " Solve 1: Standard "<< prefSeparator <<std::endl;
{
// Use an ML-style parameter list for variety
Teuchos::ParameterList MLList;
MLList.set("ML output", 10);
MLList.set("coarse: type","Amesos-Superlu");
#ifdef HAVE_AMESOS2_KLU2
MLList.set("coarse: type","Amesos-KLU");
#endif
MLParameterListInterpreter mueLuFactory(MLList);
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
Teuchos::RCP<FactoryManagerBase> LevelFactory = mueLuFactory.GetFactoryManager(1);
H->setlib(lib);
H->AddNewLevel();
H->GetLevel(1)->Keep("Nullspace",LevelFactory->GetFactory("Nullspace").get());
H->GetLevel(0)->Set("A", A);
mueLuFactory.SetupHierarchy(*H);
#ifdef HAVE_MUELU_BELOS
// Solve
MueLuExamples::solve_system_hierarchy(lib,A,X,B,H,SList);
#endif
// Extract R,P & Ac for LevelWrap Usage
H->GetLevel(1)->Get("R",R);
H->GetLevel(1)->Get("P",P);
H->GetLevel(1)->Get("A",Ac);
nullspace = H->GetLevel(1)->Get<RCP<MultiVector> >("Nullspace",LevelFactory->GetFactory("Nullspace").get());
}
out << thickSeparator << std::endl;
// =========================================================================
// Solve #2 (level wrap, the long way, using pre-done Ac)
// =========================================================================
out << thickSeparator << std::endl;
out << prefSeparator << " Solve 2: LevelWrap, Long Way, P, R, Ac "<< prefSeparator <<std::endl;
{
// Start w/ an ML-style parameter list
Teuchos::ParameterList MLList;
MLList.set("ML output", 10);
MLList.set("coarse: type","Amesos-Superlu");
#ifdef HAVE_AMESOS2_KLU2
MLList.set("coarse: type","Amesos-KLU");
#endif
FactoryManager M1;
M1.SetFactory("A", MueLu::NoFactory::getRCP());
M1.SetFactory("P", MueLu::NoFactory::getRCP());
M1.SetFactory("R", MueLu::NoFactory::getRCP());
MLParameterListInterpreter mueLuFactory(MLList);
mueLuFactory.AddFactoryManager(1, 1, Teuchos::rcpFromRef(M1));
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
H->setlib(lib);
H->GetLevel(0)->Set("A", A);
H->AddNewLevel();
H->GetLevel(1)->Set("R", R);
H->GetLevel(1)->Set("P", P);
H->GetLevel(1)->Set("A", Ac);
H->GetLevel(1)->Set("Nullspace", nullspace);
mueLuFactory.SetupHierarchy(*H);
#ifdef HAVE_MUELU_BELOS
MueLuExamples::solve_system_hierarchy(lib,A,X,B,H,SList);
#endif
}
out << thickSeparator << std::endl;
// =========================================================================
// Solve #3 (level wrap, the long way, using P, R and nullspace)
// =========================================================================
out << thickSeparator << std::endl;
out << prefSeparator << " Solve 3: LevelWrap, Long Way, P, R "<< prefSeparator <<std::endl;
{
// Start w/ an ML-style parameter list
Teuchos::ParameterList MLList;
MLList.set("ML output", 10);
MLList.set("coarse: type","Amesos-Superlu");
#ifdef HAVE_AMESOS2_KLU2
MLList.set("coarse: type","Amesos-KLU");
#endif
FactoryManager M1;
M1.SetFactory("P", MueLu::NoFactory::getRCP());
M1.SetFactory("R", MueLu::NoFactory::getRCP());
MLParameterListInterpreter mueLuFactory(MLList);
mueLuFactory.AddFactoryManager(1, 1, Teuchos::rcpFromRef(M1));
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
H->setlib(lib);
H->GetLevel(0)->Set("A", A);
H->AddNewLevel();
H->GetLevel(1)->Set("R", R);
H->GetLevel(1)->Set("P", P);
H->GetLevel(1)->Set("Nullspace", nullspace);
mueLuFactory.SetupHierarchy(*H);
#ifdef HAVE_MUELU_BELOS
MueLuExamples::solve_system_hierarchy(lib,A,X,B,H,SList);
#endif
}
out << thickSeparator << std::endl;
// =========================================================================
// Solve #4 (level wrap, the fast way, everything)
// =========================================================================
out << thickSeparator << std::endl;
out << prefSeparator << " Solve 4: LevelWrap, Fast Way, P, R, Ac "<< prefSeparator <<std::endl;
{
Teuchos::ParameterList MueLuList, level1;
level1.set("A",Ac);
level1.set("R",R);
level1.set("P",P);
level1.set("Nullspace",nullspace);
MueLuList.set("level 1",level1);
MueLuList.set("verbosity","high");
MueLuList.set("coarse: max size",100);
MueLuList.set("max levels",4);
#ifdef HAVE_MUELU_BELOS
MueLuExamples::solve_system_list(lib,A,X,B,MueLuList,SList);
#endif
}
// =========================================================================
// Solve #5 (level wrap, the fast way, P, R + Nullspace)
// =========================================================================
out << thickSeparator << std::endl;
out << prefSeparator << " Solve 5: LevelWrap, Fast Way, P, R "<< prefSeparator <<std::endl;
{
Teuchos::ParameterList MueLuList, level1;
level1.set("R",R);
level1.set("P",P);
level1.set("Nullspace",nullspace);
MueLuList.set("level 1",level1);
MueLuList.set("verbosity","high");
MueLuList.set("coarse: max size",100);
#ifdef HAVE_MUELU_BELOS
MueLuExamples::solve_system_list(lib,A,X,B,MueLuList,SList);
#endif
}
// =========================================================================
// Solve #6 (level wrap, the fast way, P only, explicit transpose)
// =========================================================================
out << thickSeparator << std::endl;
out << prefSeparator << " Solve 6: LevelWrap, Fast Way, P only, explicit transpose "<< prefSeparator <<std::endl;
{
Teuchos::ParameterList MueLuList, level1;
level1.set("P",P);
level1.set("Nullspace",nullspace);
MueLuList.set("level 1",level1);
MueLuList.set("verbosity","high");
MueLuList.set("coarse: max size",100);
MueLuList.set("transpose: use implicit",false);
MueLuList.set("max levels",4);
#ifdef HAVE_MUELU_BELOS
MueLuExamples::solve_system_list(lib,A,X,B,MueLuList,SList);
#endif
}
// =========================================================================
// Solve #7 (level wrap, the fast way, P only, implicit transpose)
// =========================================================================
out << thickSeparator << std::endl;
out << prefSeparator << " Solve 7: LevelWrap, Fast Way, P only, implicit transpose "<< prefSeparator <<std::endl;
{
Teuchos::ParameterList MueLuList, level1;
level1.set("P",P);
level1.set("Nullspace",nullspace);
MueLuList.set("level 1",level1);
MueLuList.set("verbosity","high");
MueLuList.set("coarse: max size",100);
MueLuList.set("transpose: use implicit",true);
MueLuList.set("max levels",2);
#ifdef HAVE_MUELU_BELOS
MueLuExamples::solve_system_list(lib,A,X,B,MueLuList,SList);
#endif
}
success = true;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main(int argc, char *argv[]) {
using Teuchos::RCP;
using Teuchos::rcp;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
*out << MueLu::MemUtils::PrintMemoryUsage() << std::endl;
// Timing
Teuchos::Time myTime("global");
Teuchos::TimeMonitor M(myTime);
#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
#endif
int globalNumDofs = 7020; //3402;
int nProcs = comm->getSize();
int nDofsPerNode = 2;
int nLocalDofs = (int) globalNumDofs / nProcs;
nLocalDofs = nLocalDofs - (nLocalDofs % nDofsPerNode);
int nCumulatedDofs = 0;
sumAll(comm,nLocalDofs, nCumulatedDofs);
if(comm->getRank() == nProcs-1) {
nLocalDofs += globalNumDofs - nCumulatedDofs;
}
std::cout << "PROC: " << comm->getRank() << " numLocalDofs=" << nLocalDofs << std::endl;
// read in problem
Epetra_Map emap (globalNumDofs, nLocalDofs, 0, *Xpetra::toEpetra(comm));
//Epetra_Map emap(3402,0,*Xpetra::toEpetra(comm));
Epetra_CrsMatrix * ptrA = 0;
Epetra_Vector * ptrf = 0;
Epetra_MultiVector* ptrNS = 0;
std::cout << "Reading matrix market file" << std::endl;
/*EpetraExt::MatrixMarketFileToCrsMatrix("/home/tobias/trilinos/Trilinos_dev/ubuntu_openmpi/preCopyrightTrilinos/muelu/example/Structure/stru2d_A.txt",emap,emap,emap,ptrA);
EpetraExt::MatrixMarketFileToVector("/home/tobias/trilinos/Trilinos_dev/ubuntu_openmpi/preCopyrightTrilinos/muelu/example/Structure/stru2d_b.txt",emap,ptrf);
EpetraExt::MatrixMarketFileToMultiVector( "/home/tobias/trilinos/Trilinos_dev/ubuntu_openmpi/preCopyrightTrilinos/muelu/example/Structure/stru2d_ns.txt", emap, ptrNS);*/
/*EpetraExt::MatrixMarketFileToCrsMatrix("/home/wiesner/trilinos/Trilinos_dev/fc8_openmpi_dbg_q52011/preCopyrightTrilinos/muelu/example/Structure/stru2d_A.txt",emap,emap,emap,ptrA);
EpetraExt::MatrixMarketFileToVector("/home/wiesner/trilinos/Trilinos_dev/fc8_openmpi_dbg_q52011/preCopyrightTrilinos/muelu/example/Structure/stru2d_b.txt",emap,ptrf);
EpetraExt::MatrixMarketFileToMultiVector( "/home/wiesner/trilinos/Trilinos_dev/fc8_openmpi_dbg_q52011/preCopyrightTrilinos/muelu/example/Structure/stru2d_ns.txt", emap, ptrNS);*/
EpetraExt::MatrixMarketFileToCrsMatrix("stru2d_A.txt",emap,emap,emap,ptrA);
EpetraExt::MatrixMarketFileToVector("stru2d_b.txt",emap,ptrf);
EpetraExt::MatrixMarketFileToMultiVector( "stru2d_ns.txt", emap, ptrNS);
RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
RCP<Epetra_Vector> epv = Teuchos::rcp(ptrf);
RCP<Epetra_MultiVector> epNS = Teuchos::rcp(ptrNS);
// Epetra_CrsMatrix -> Xpetra::Matrix
RCP<CrsMatrix> exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(epA));
RCP<CrsMatrixWrap> crsOp = Teuchos::rcp(new CrsMatrixWrap(exA));
RCP<Matrix> Op = crsOp;
// Epetra_Vector -> Xpetra::Vector
RCP<Vector> xRhs = Teuchos::rcp(new Xpetra::EpetraVector(epv));
RCP<MultiVector> xNS = Teuchos::rcp(new Xpetra::EpetraMultiVector(epNS));
// Epetra_Map -> Xpetra::Map
const RCP< const Map> map = Xpetra::toXpetra(emap);
Teuchos::ParameterList mlParams;
FillMLParameterList(mlParams); // fill ML parameter list (without nullspace)
MLParameterListInterpreter mueLuFactory(mlParams);
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", Op);
H->GetLevel(0)->Set("Nullspace", xNS);
mueLuFactory.SetupHierarchy(*H);
H->SetVerbLevel(MueLu::High);
RCP<MultiVector> xLsg = MultiVectorFactory::Build(map,1);
// Use AMG directly as an iterative method
{
xLsg->putScalar( (SC) 0.0);
H->Iterate(*xRhs,10,*xLsg);
//xLsg->describe(*out,Teuchos::VERB_EXTREME);
}
//
// Solve Ax = b using AMG as a preconditioner in AztecOO
//
{
RCP<Epetra_Vector> X = rcp(new Epetra_Vector(epv->Map()));
X->PutScalar(0.0);
Epetra_LinearProblem epetraProblem(epA.get(), X.get(), epv.get());
AztecOO aztecSolver(epetraProblem);
aztecSolver.SetAztecOption(AZ_solver, AZ_cg);
MueLu::EpetraOperator aztecPrec(H);
aztecSolver.SetPrecOperator(&aztecPrec);
int maxIts = 50;
double tol = 1e-8;
aztecSolver.Iterate(maxIts, tol);
}
/*for(int i=0; i<H->GetNumLevels(); i++) {
RCP<Level> l = H->GetLevel(i);
*out << std::endl << "Level " << i << std::endl;
l->print(*out);
}*/
return EXIT_SUCCESS;
}
