int main(int argc, char** argv)
{
using Teuchos::RCP;
typedef int LO; // LocalOrdinal
typedef int GO; // GlobalOrdinal
typedef double SC; // Scalar
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
/**********************************************************************************/
/* SET TEST PARAMETERS */
/**********************************************************************************/
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
Galeri::Xpetra::Parameters<GO> matrixParameters(clp); // manage parameters of the test case
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
matrixParameters.check();
std::cout << matrixParameters;
/**********************************************************************************/
/* CREATE INITAL MATRIX */
/**********************************************************************************/
RCP<const Tpetra::Map<LO,GO> > map = Teuchos::rcp( new Tpetra::Map<LO,GO>(matrixParameters.GetNumGlobalElements(), 0, comm) );
RCP<Galeri::Xpetra::Problem<Tpetra::Map<LO,GO>,Tpetra::CrsMatrix<SC,LO,GO>,Tpetra::MultiVector<SC,LO,GO> > > problem =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Tpetra::Map<LO,GO>, Tpetra::CrsMatrix<SC,LO,GO>, Tpetra::MultiVector<SC,LO,GO> >
(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Tpetra::CrsMatrix<SC,LO,GO> > A = problem->BuildMatrix();
/**********************************************************************************/
/* */
/**********************************************************************************/
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
if (comm->getRank() == 0)
std::cout << "\n================ MAP =====================================================\n" << std::endl;
map->describe(*out, Teuchos::VERB_EXTREME);
comm->barrier();
#ifdef _MSC_VER
Sleep(1000);
#else
sleep(1);
#endif
if (comm->getRank() == 0)
std::cout << "\n================ MATRIX ==================================================\n" << std::endl;
A->describe(*out, Teuchos::VERB_EXTREME);
return EXIT_SUCCESS;
}
void generate_user_matrix_and_nullspace(std::string &matrixType, Xpetra::UnderlyingLib & lib,Teuchos::ParameterList &galeriList, RCP<const Teuchos::Comm<int> > &comm, RCP<Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > & A, RCP<Xpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> > & nullspace){
using Teuchos::RCP;
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& out = *fancy;
typedef typename Xpetra::Map<LocalOrdinal,GlobalOrdinal,Node> map_type;
typedef typename Xpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> multivector_type;
typedef typename Xpetra::CrsMatrixWrap<Scalar,LocalOrdinal,GlobalOrdinal,Node> matrixwrap_type;
RCP<const map_type > map;
RCP<multivector_type > coordinates;
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LocalOrdinal, GlobalOrdinal, Node>(lib, "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<Scalar,LocalOrdinal,GlobalOrdinal,map_type,multivector_type>("1D", map, galeriList);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" || matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LocalOrdinal, GlobalOrdinal, Node>(lib, "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<Scalar,LocalOrdinal,GlobalOrdinal,map_type,multivector_type>("2D", map, galeriList);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LocalOrdinal, GlobalOrdinal, Node>(lib, "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<Scalar,LocalOrdinal,GlobalOrdinal,map_type,multivector_type>("3D", map, galeriList);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LocalOrdinal,GlobalOrdinal,Node>::Build(map, (matrixType == "Elasticity2D" ? 2 : 3));
out << "Processor subdomains in x direction: " << galeriList.get<GlobalOrdinal>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriList.get<GlobalOrdinal>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriList.get<GlobalOrdinal>("mz") << std::endl
<< "========================================================" << std::endl;
RCP<Galeri::Xpetra::Problem<map_type,matrixwrap_type,multivector_type> > Pr =
Galeri::Xpetra::BuildProblem<Scalar,LocalOrdinal,GlobalOrdinal,map_type,matrixwrap_type,multivector_type>(matrixType, map, galeriList);
A = Pr->BuildMatrix();
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((matrixType == "Elasticity2D") ? 2 : 3);
}
}
TEUCHOS_UNIT_TEST(GenericRFactory, SymmetricProblem)
{
out << "version: " << MueLu::Version() << std::endl;
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// generate problem
LO maxLevels = 3;
LO nEle = 63;
const RCP<const Map> map = MapFactory::Build(TestHelpers::Parameters::getLib(), nEle, 0, comm);
Teuchos::ParameterList matrixParameters;
matrixParameters.set("nx",nEle);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Map, CrsMatrixWrap, MultiVector>("Laplace1D", map, matrixParameters);
RCP<Matrix> Op = Pr->BuildMatrix();
// build nullspace
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
out << "||NS|| = " << norms[0] << std::endl;
// fill hierarchy
RCP<Hierarchy> H = rcp( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
RCP<Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op); // set fine level matrix
Finest->Set("Nullspace",nullSpace); // set null space information for finest level
// define transfer operators
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetMinNodesPerAggregate(3);
CoupledAggFact->SetMaxNeighAlreadySelected(0);
CoupledAggFact->SetOrdering("natural");
CoupledAggFact->SetPhase3AggCreation(0.5);
RCP<SaPFactory> Pfact = rcp( new SaPFactory());
RCP<Factory> Rfact = rcp( new GenericRFactory() );
H->SetMaxCoarseSize(1);
// setup smoothers
Teuchos::ParameterList smootherParamList;
smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
smootherParamList.set("relaxation: sweeps", (LO) 1);
smootherParamList.set("relaxation: damping factor", (SC) 1.0);
RCP<SmootherPrototype> smooProto = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );
RCP<SmootherFactory> SmooFact = rcp( new SmootherFactory(smooProto) );
//Acfact->setVerbLevel(Teuchos::VERB_HIGH);
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(smooProto, Teuchos::null));
FactoryManager M;
M.SetFactory("P", Pfact);
M.SetFactory("R", Rfact);
M.SetFactory("Aggregates", CoupledAggFact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", coarseSolveFact);
H->Setup(M, 0, maxLevels);
RCP<Level> coarseLevel = H->GetLevel(1);
RCP<Matrix> P1 = coarseLevel->Get< RCP<Matrix> >("P");
RCP<Matrix> R1 = coarseLevel->Get< RCP<Matrix> >("R");
RCP<Level> coarseLevel2 = H->GetLevel(2);
RCP<Matrix> P2 = coarseLevel2->Get< RCP<Matrix> >("P");
RCP<Matrix> R2 = coarseLevel2->Get< RCP<Matrix> >("R");
TEST_EQUALITY(Finest->IsAvailable("PreSmoother"), true);
TEST_EQUALITY(Finest->IsAvailable("PostSmoother"), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PreSmoother"), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PostSmoother"), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PreSmoother"), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PostSmoother"), false);
// test some basic multgrid data
TEST_EQUALITY(P1->getGlobalNumEntries(), R1->getGlobalNumEntries());
TEST_EQUALITY(P1->getGlobalNumRows(), R1->getGlobalNumCols());
TEST_EQUALITY(P1->getGlobalNumCols(), R1->getGlobalNumRows());
TEST_EQUALITY(P2->getGlobalNumEntries(), R2->getGlobalNumEntries());
TEST_EQUALITY(P2->getGlobalNumRows(), R2->getGlobalNumCols());
TEST_EQUALITY(P2->getGlobalNumCols(), R2->getGlobalNumRows());
//RCP<Teuchos::FancyOStream> fos = getFancyOStream(Teuchos::rcpFromRef(cout));
// since A is chosen symmetric, it is P^T = R
// check P^T * P = R * P
// note: the Epetra matrix-matrix multiplication using implicit transpose is buggy in parallel case
// (for multiplication of a square matrix with a rectangular matrix)
// however it seems to work for two rectangular matrices
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > RP = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*R1,false,*P1,false,out);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > PtP = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*P1,true,*P1,false,out);
RCP<Vector> x = VectorFactory::Build(RP->getDomainMap());
RCP<Vector> bRP = VectorFactory::Build(RP->getRangeMap());
RCP<Vector> bPtP = VectorFactory::Build(PtP->getRangeMap());
x->randomize();
RP->apply(*x,*bRP);
PtP->apply(*x,*bPtP);
TEST_EQUALITY(bRP->norm1() - bPtP->norm1() < 1e-12, true);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > RP2 = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*R2,false,*P2,false,out);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO,Node> > PtP2 = Xpetra::MatrixMatrix<Scalar,LO,GO,Node>::Multiply(*P2,true,*P2,false,out);
x = VectorFactory::Build(RP2->getDomainMap());
bRP = VectorFactory::Build(RP2->getRangeMap());
bPtP = VectorFactory::Build(PtP2->getRangeMap());
x->randomize();
RP2->apply(*x,*bRP);
PtP2->apply(*x,*bPtP);
TEST_EQUALITY(bRP->norm1() - bPtP->norm1() < 1e-12, true);
//R1->describe(*fos,Teuchos::VERB_EXTREME);
}
int main(int argc, char *argv[]) {
// Define default types
typedef double scalar_type;
typedef int local_ordinal_type;
typedef int global_ordinal_type;
typedef KokkosClassic::DefaultNode::DefaultNodeType node_type;
// Convenient typedef's
typedef Tpetra::Operator<scalar_type,local_ordinal_type,global_ordinal_type,node_type> operator_type;
typedef Tpetra::CrsMatrix<scalar_type,local_ordinal_type,global_ordinal_type,node_type> crs_matrix_type;
typedef Tpetra::RowMatrix<scalar_type,local_ordinal_type,global_ordinal_type,node_type> row_matrix_type;
typedef Tpetra::Vector<scalar_type,local_ordinal_type,global_ordinal_type,node_type> vector_type;
typedef Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_type,node_type> multivector_type;
typedef Tpetra::Map<local_ordinal_type,global_ordinal_type,node_type> driver_map_type;
typedef MueLu::TpetraOperator<scalar_type, local_ordinal_type, global_ordinal_type, node_type> muelu_tpetra_operator_type;
typedef MueLu::Utilities<scalar_type,local_ordinal_type,global_ordinal_type,node_type> MueLuUtilities;
typedef Belos::LinearProblem<scalar_type, multivector_type, operator_type> linear_problem_type;
typedef Belos::SolverManager<scalar_type, multivector_type, operator_type> belos_solver_manager_type;
typedef Belos::PseudoBlockCGSolMgr<scalar_type, multivector_type, operator_type> belos_pseudocg_manager_type;
typedef Belos::BlockGmresSolMgr<scalar_type, multivector_type, operator_type> belos_gmres_manager_type;
typedef Belos::BiCGStabSolMgr<scalar_type, multivector_type, operator_type> belos_bicgstab_manager_type;
typedef Ifpack2::Preconditioner<scalar_type, local_ordinal_type, global_ordinal_type, node_type> precond_type;
//MueLu_UseShortNames.hpp wants these typedefs.
typedef scalar_type Scalar;
typedef local_ordinal_type LocalOrdinal;
typedef global_ordinal_type GlobalOrdinal;
typedef node_type Node;
# include <MueLu_UseShortNames.hpp>
typedef Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> GaleriXpetraProblem;
typedef ADR::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> ADRXpetraProblem;
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp; // reference count pointers
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int mypid = comm->getRank();
/*
int subCommRank[3]={0,1,2};
Teuchos::ArrayView<int> arraySubCommRank(subCommRank, 3);
auto subComm = comm->createSubcommunicator(arraySubCommRank);
*/
Teuchos::CommandLineProcessor clp(false);
global_ordinal_type maxIts = 10000;
scalar_type tol = 1e-10;
std::string solverOptionsFile = "final_parser.xml";
std::string krylovSolverType = "bicgstab";
clp.setOption("xmlFile", &solverOptionsFile, "XML file containing MueLu solver parameters");
clp.setOption("maxits", &maxIts, "maximum number of Krylov iterations");
clp.setOption("tol", &tol, "tolerance for Krylov solver");
clp.setOption("krylovType", &krylovSolverType, "cg or gmres solver");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
ParameterList xmlParams;
ParameterList mueluParams;
ParameterList problemParams;
Teuchos::updateParametersFromXmlFile(solverOptionsFile, Teuchos::inoutArg(xmlParams));
mueluParams = xmlParams.sublist(static_cast<const std::string>("MueLu"));
problemParams = xmlParams.sublist(static_cast<const std::string>("Problem"));
// Problem definition
std::string problem_type = problemParams.get<std::string>(static_cast<const std::string>("problem type"));
// Parameters
Scalar Lx = problemParams.get<scalar_type>(static_cast<const std::string>("domain size in x-direction"));
Scalar Ly = problemParams.get<scalar_type>(static_cast<const std::string>("domain size in y-direction"));
Scalar Lz = problemParams.get<scalar_type>(static_cast<const std::string>("domain size in z-direction"));
global_ordinal_type nx = problemParams.get<int>(static_cast<const std::string>("nodes in x-direction"));
global_ordinal_type ny = problemParams.get<int>(static_cast<const std::string>("nodes in y-direction"));
global_ordinal_type nz = problemParams.get<int>(static_cast<const std::string>("nodes in z-direction"));
global_ordinal_type number_runs = problemParams.get<int>(static_cast<const std::string>("number of runs"));
MueLu::DomainPartitioning domain;
int keep_boundary = 0;
Scalar stretchx = (Scalar) Lx/nx;
Scalar stretchy = (Scalar) Ly/ny;
Scalar stretchz = (Scalar) Lz/nz;
ADR::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, problem_type, keep_boundary , stretchx, stretchy, stretchz); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
//
// Construct the problem
//
global_ordinal_type indexBase = 0;
RCP<const Map> xpetraMap;
std::vector<global_ordinal_type> ind;
//BRICK SIZE
int brick_sizex = mueluParams.get<int>(static_cast<const std::string>("aggregation: brick x size"));
int brick_sizey = mueluParams.get<int>(static_cast<const std::string>("aggregation: brick y size"));
int brick_sizez = mueluParams.get<int>(static_cast<const std::string>("aggregation: brick z size"));
//Creation of the map where processor 0 gets nothing at the fine level
if( comm->getSize()>1 )
{
if(problem_type == "ADR1D")
createBrickMap1D(matrixParameters.GetNumGlobalElements(), ind, comm );
else if(problem_type == "ADR2D")
createBrickMap2D( nx, brick_sizex, brick_sizey, ind, comm );
else if(problem_type == "ADR3D")
createBrickMap3D( nx, ny, brick_sizex, brick_sizey, brick_sizez, ind, comm );
ind.shrink_to_fit();
Teuchos::ArrayView<const global_ordinal_type> elementList (ind);
xpetraMap = MapFactory::Build(Xpetra::UseTpetra,matrixParameters.GetNumGlobalElements(), elementList, indexBase, comm);
}
else if( comm->getSize()==1 )
xpetraMap = MapFactory::Build(Xpetra::UseTpetra, matrixParameters.GetNumGlobalElements(), indexBase, comm);
RCP<MultiVector> coordinates;
if (problem_type == "ADR1D")
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<scalar_type,local_ordinal_type,global_ordinal_type,Map,MultiVector>("1D", xpetraMap, matrixParameters.GetParameterList());
else if (problem_type == "ADR2D")
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<scalar_type,local_ordinal_type,global_ordinal_type,Map,MultiVector>("2D", xpetraMap, matrixParameters.GetParameterList());
else if (problem_type == "ADR3D")
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<scalar_type,local_ordinal_type,global_ordinal_type,Map,MultiVector>("3D", xpetraMap, matrixParameters.GetParameterList());
RCP<ADRXpetraProblem> Pr = ADR::Xpetra::BuildProblem<scalar_type, local_ordinal_type, global_ordinal_type, Map, CrsMatrixWrap, MultiVector>(matrixParameters.GetMatrixType(), xpetraMap, matrixParameters.GetParameterList());
RCP<Matrix> xpetraA = Pr->BuildMatrix();
RCP<crs_matrix_type> A = MueLuUtilities::Op2NonConstTpetraCrs(xpetraA);
RCP<const driver_map_type> map = MueLuUtilities::Map2TpetraMap(*xpetraMap);
// ===================================================
// Domain Decomposition Preconditioner
// ===================================
//Creation of the MueLu list for the DD preconditioner
RCP<ParameterList> dd_list = rcp(new Teuchos::ParameterList());
dd_list->setName("MueLu");
dd_list->set("verbosity", "low");
dd_list->set("number of equations", 1);
dd_list->set("max levels", 1);
dd_list->set("coarse: type", "SCHWARZ"); //FOR A ONE LEVEL PRECONDITIONER THE COARSE LEVEL IS INTERPRETED AS SMOOTHING LEVEL
ParameterList& dd_smooth_sublist = dd_list->sublist("coarse: params");
dd_smooth_sublist.set("schwarz: overlap level", 0);
dd_smooth_sublist.set("schwarz: combine mode", "Zero");
dd_smooth_sublist.set("subdomain solver name", "RILUK");
ParameterList& coarse_subdomain_solver = dd_smooth_sublist.sublist("subdomain solver parameters");
coarse_subdomain_solver.set("fact: iluk level-of-fill", 3);
coarse_subdomain_solver.set("fact: absolute threshold", 0.);
coarse_subdomain_solver.set("fact: relative threshold", 1.);
coarse_subdomain_solver.set("fact: relax value", 0.);
RCP<muelu_tpetra_operator_type> B_DD = MueLu::CreateTpetraPreconditioner( (RCP<operator_type>)A, *dd_list );
// ===================================================
// Multi Grid Preconditioner
// ===================================
RCP<muelu_tpetra_operator_type> M;
M = MueLu::CreateTpetraPreconditioner( (RCP<operator_type>)A, mueluParams, Utilities::MV2NonConstTpetraMV(coordinates) );
RCP<multivector_type> X_muelu = rcp(new multivector_type(map,1));
RCP<multivector_type> B = rcp(new multivector_type(map,1));
RCP<linear_problem_type> Problem_muelu;
X_muelu->putScalar((scalar_type) 0.0);
B->randomize();
Problem_muelu = rcp(new linear_problem_type(A, X_muelu, B));
RCP<ParameterList> belosList = rcp(new ParameterList());
belosList->set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList->set("Convergence Tolerance", tol); // Relative convergence tolerance requested
//belosList->set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList->set("Verbosity", Belos::Errors);
belosList->set("Output Frequency", 1);
belosList->set("Output Style", Belos::Brief);
belosList->set("Implicit Residual Scaling", "None");
RCP<belos_solver_manager_type> solver;
if (krylovSolverType == "cg")
solver = rcp(new belos_pseudocg_manager_type(Problem_muelu, belosList));
else if (krylovSolverType == "gmres")
solver = rcp(new belos_gmres_manager_type(Problem_muelu, belosList));
else if (krylovSolverType == "bicgstab")
solver = rcp(new belos_bicgstab_manager_type(Problem_muelu, belosList));
else
throw std::invalid_argument("bad Krylov solver type");
for(int trial = 1; trial<=number_runs; ++trial)
{
X_muelu->putScalar((scalar_type) 0.0);
B->randomize();
//
// Set up Krylov solver and iterate.
//
Problem_muelu = rcp(new linear_problem_type(A, X_muelu, B));
Problem_muelu->setRightPrec(M);
Problem_muelu->setProblem();
solver->setProblem(Problem_muelu);
solver->solve();
int numIterations_muelu = solver->getNumIters();
Teuchos::Array<typename Teuchos::ScalarTraits<scalar_type>::magnitudeType> normVec_muelu(1);
multivector_type residual_muelu(B->getMap(),1);
A->apply(*X_muelu, residual_muelu);
residual_muelu.update(1.0, *B, -1.0);
residual_muelu.norm2(normVec_muelu);
if (mypid == 0) {
std::cout << "number of iterations with MueLu preconditioner= " << numIterations_muelu << std::endl;
std::cout << "||Residual|| = " << normVec_muelu[0] << std::endl;
}
}
#include <Teuchos_TimeMonitor.hpp>
Teuchos::TimeMonitor::summarize ();
return EXIT_SUCCESS;
}
int main_(Teuchos::CommandLineProcessor &clp, Xpetra::UnderlyingLib lib, int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
//
// MPI initialization
//
Teuchos::oblackholestream blackhole;
bool success = false;
bool verbose = true;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Process command line arguments
//
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 81); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
default:;
}
if (comm->getRank() == 0) std::cout << xpetraParameters << matrixParameters;
//
// Setup test case (Ax = b)
//
// Distribution
RCP<const Map> map = MapFactory::Build(lib, matrixParameters.GetNumGlobalElements(), 0, comm);
// Matrix
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Map, CrsMatrixWrap, MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
// User defined nullspace
RCP<MultiVector> nullSpace = VectorFactory::Build(map,1); nullSpace->putScalar((SC) 1.0);
// Define B
RCP<Vector> X = VectorFactory::Build(map,1);
RCP<Vector> B = VectorFactory::Build(map,1);
X->setSeed(846930886);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
// X = 0
X->putScalar((SC) 0.0);
//
// Create a multigrid configuration
//
// Transfer operators
RCP<TentativePFactory> TentativePFact = rcp( new TentativePFactory() );
RCP<SaPFactory> SaPFact = rcp( new SaPFactory() );
RCP<TransPFactory> RFact = rcp( new TransPFactory());
FactoryManager M;
M.SetFactory("Ptent", TentativePFact);
M.SetFactory("P", SaPFact);
M.SetFactory("R", RFact);
M.SetFactory("Smoother", Teuchos::null); //skips smoother setup
M.SetFactory("CoarseSolver", Teuchos::null); //skips coarsest solve setup
//
// Multigrid setup phase
//
int startLevel = 0;
int maxLevels = 10;
std::cout << "=============== Setup transfers only ====================" << std::endl;
Hierarchy H;
H.SetDefaultVerbLevel(MueLu::Medium);
RCP<Level> finestLevel = H.GetLevel();
finestLevel->Set("A", A);
finestLevel->Set("Nullspace", nullSpace);
// Indicate which Hierarchy operators we want to keep
H.Keep("P", SaPFact.get()); //SaPFact is the generating factory for P.
H.Keep("R", RFact.get()); //RFact is the generating factory for R.
H.Keep("Ptent", TentativePFact.get()); //SaPFact is the generating factory for P.
H.Setup(M,startLevel,maxLevels);
std::cout << "=============== Setup smoothers only ====================" << std::endl;
// Create a new A.
RCP<Matrix> newA = Pr->BuildMatrix();
finestLevel->Set("A", newA);
// Create Gauss-Seidel smoother.
std::string ifpackType = "RELAXATION";
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) 3);
ifpackList.set("relaxation: damping factor", (SC) 1.0);
RCP<SmootherPrototype> smootherPrototype = rcp(new TrilinosSmoother(ifpackType, ifpackList));
M.SetFactory("Smoother", rcp(new SmootherFactory(smootherPrototype)));
// Create coarsest solver.
RCP<SmootherPrototype> coarseSolverPrototype = rcp( new DirectSolver() );
RCP<SmootherFactory> coarseSolverFact = rcp( new SmootherFactory(coarseSolverPrototype, Teuchos::null) );
M.SetFactory("CoarseSolver", coarseSolverFact);
// Note that we pass the number of levels back in.
H.Setup(M,startLevel, H.GetNumLevels());
std::cout << "=============== Solve ====================" << std::endl;
//
// Solve Ax = B
//
LO nIts = 9;
H.Iterate(*B, *X, nIts);
//
// Print relative residual norm
//
typename Teuchos::ScalarTraits<SC>::magnitudeType residualNorms = Utilities::ResidualNorm(*A, *X, *B)[0];
if (comm->getRank() == 0) {
std::ios::fmtflags f(std::cout.flags());
std::cout << "||Residual|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(20) << residualNorms << std::endl;
std::cout.flags(f);
}
success = true;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
TEUCHOS_UNIT_TEST(SaPFactory_kokkos, EpetraVsTpetra)
{
# include "MueLu_UseShortNames.hpp"
MueLu::VerboseObject::SetDefaultOStream(Teuchos::rcpFromRef(out));
out << "version: " << MueLu::Version() << std::endl;
out << "Compare results of Epetra and Tpetra" << std::endl;
out << "for 3 level AMG solver using smoothed aggregation with" << std::endl;
out << "one SGS sweep on each multigrid level as pre- and postsmoother" << std::endl;
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
Array<STS::magnitudeType> results(2);
// run test only on 1 proc
if(comm->getSize() == 1) {
Xpetra::UnderlyingLib lib = Xpetra::UseEpetra;
// run Epetra and Tpetra test
for (int run = 0; run < 2; run++) { //TODO: create a subfunction instead or Tuple of UnderlyingLib
if (run == 0) lib = Xpetra::UseEpetra;
else lib = Xpetra::UseTpetra;
// generate problem
LO maxLevels = 3;
LO its = 10;
GO nEle = 63;
const RCP<const Map> map = MapFactory::Build(lib, nEle, 0, comm);
Teuchos::ParameterList matrixParameters;
matrixParameters.set("nx", nEle);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>("Laplace1D", map, matrixParameters);
RCP<Matrix> Op = Pr->BuildMatrix();
// build nullspace
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar(one);
Array<STS::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
out << "||NS|| = " << norms[0] << std::endl;
// fill hierarchy
RCP<Hierarchy> H = rcp( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
RCP<Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op); // set fine level matrix
Finest->Set("Nullspace",nullSpace); // set null space information for finest level
// define transfer operators
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetMinNodesPerAggregate(3);
CoupledAggFact->SetMaxNeighAlreadySelected(0);
CoupledAggFact->SetOrdering("natural");
CoupledAggFact->SetPhase3AggCreation(0.5);
RCP<TentativePFactory> Ptentfact = rcp(new TentativePFactory());
RCP<SaPFactory> Pfact = rcp( new SaPFactory());
RCP<Factory> Rfact = rcp( new TransPFactory() );
RCP<RAPFactory> Acfact = rcp( new RAPFactory() );
H->SetMaxCoarseSize(1);
// setup smoothers
Teuchos::ParameterList smootherParamList;
smootherParamList.set("relaxation: type", "Symmetric Gauss-Seidel");
smootherParamList.set("relaxation: sweeps", (LO) 1);
smootherParamList.set("relaxation: damping factor", (SC) 1.0);
RCP<SmootherPrototype> smooProto = rcp( new TrilinosSmoother("RELAXATION", smootherParamList) );
RCP<SmootherFactory> SmooFact = rcp( new SmootherFactory(smooProto) );
Acfact->setVerbLevel(Teuchos::VERB_HIGH);
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(smooProto, Teuchos::null));
FactoryManager M;
M.SetFactory("P", Pfact);
M.SetFactory("R", Rfact);
M.SetFactory("A", Acfact);
M.SetFactory("Ptent", Ptentfact);
M.SetFactory("Aggregates", CoupledAggFact);
M.SetFactory("Smoother", SmooFact);
M.SetFactory("CoarseSolver", coarseSolveFact);
H->Setup(M, 0, maxLevels);
// test some basic multigrid data
RCP<Level> coarseLevel = H->GetLevel(1);
TEST_EQUALITY(coarseLevel->IsRequested("A",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("P",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PreSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PostSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("R",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("A",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("P",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PreSmoother",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("PostSmoother",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->IsAvailable("R",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel->GetKeepFlag("A",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("P",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PreSmoother",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PostSmoother",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->GetKeepFlag("R",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel->IsRequested("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PreSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("PostSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel->IsRequested("A",Acfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("PreSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("PostSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel->IsAvailable("A",Acfact.get()), false);
TEST_EQUALITY(coarseLevel->GetKeepFlag("P",Pfact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("P",Ptentfact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PreSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("PostSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("R",Rfact.get()), 0);
TEST_EQUALITY(coarseLevel->GetKeepFlag("A",Acfact.get()), 0);
RCP<Matrix> P1 = coarseLevel->Get< RCP<Matrix> >("P");
RCP<Matrix> R1 = coarseLevel->Get< RCP<Matrix> >("R");
TEST_EQUALITY(P1->getGlobalNumRows(), 63);
TEST_EQUALITY(P1->getGlobalNumCols(), 21);
TEST_EQUALITY(R1->getGlobalNumRows(), 21);
TEST_EQUALITY(R1->getGlobalNumCols(), 63);
RCP<Level> coarseLevel2 = H->GetLevel(2);
TEST_EQUALITY(coarseLevel2->IsRequested("A",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("P",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("R",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("PreSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("PostSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("A",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("P",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PreSmoother",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->IsAvailable("PostSmoother",MueLu::NoFactory::get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("R",MueLu::NoFactory::get()), true);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("A",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("P",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PreSmoother",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PostSmoother",MueLu::NoFactory::get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("R",MueLu::NoFactory::get()), MueLu::Final);
TEST_EQUALITY(coarseLevel2->IsRequested("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsRequested("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("P",Pfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("P",Ptentfact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("PreSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("PostSmoother",SmooFact.get()), false);
TEST_EQUALITY(coarseLevel2->IsAvailable("R",Rfact.get()), false);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("P",Pfact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("P",Ptentfact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PreSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("PostSmoother",SmooFact.get()), 0);
TEST_EQUALITY(coarseLevel2->GetKeepFlag("R",Rfact.get()), 0);
RCP<Matrix> P2 = coarseLevel2->Get< RCP<Matrix> >("P");
RCP<Matrix> R2 = coarseLevel2->Get< RCP<Matrix> >("R");
TEST_EQUALITY(P2->getGlobalNumRows(), 21);
TEST_EQUALITY(P2->getGlobalNumCols(), 7);
TEST_EQUALITY(R2->getGlobalNumRows(), 7);
TEST_EQUALITY(R2->getGlobalNumCols(), 21);
Teuchos::RCP<Xpetra::Matrix<Scalar,LO,GO> > PtentTPtent = Xpetra::MatrixMatrix<Scalar,LO,GO>::Multiply(*P1,true,*P1,false,out);
TEST_EQUALITY(PtentTPtent->getGlobalMaxNumRowEntries()-3<1e-12, true);
TEST_EQUALITY(P1->getGlobalMaxNumRowEntries()-2<1e-12, true);
TEST_EQUALITY(P2->getGlobalMaxNumRowEntries()-2<1e-12, true);
// Define RHS
RCP<MultiVector> X = MultiVectorFactory::Build(map,1);
RCP<MultiVector> RHS = MultiVectorFactory::Build(map,1);
X->putScalar(1.0);
X->norm2(norms);
if (comm->getRank() == 0)
out << "||X_true|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
Op->apply(*X,*RHS,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
// Use AMG directly as an iterative method
{
X->putScalar( (SC) 0.0);
H->Iterate(*RHS,*X,its);
X->norm2(norms);
if (comm->getRank() == 0)
out << "||X_" << std::setprecision(2) << its << "|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
results[run] = norms[0];
}
}
TEST_EQUALITY(results[0] - results[1] < 1e-10, true); // check results of EPETRA vs TPETRA
} // comm->getSize == 1
} //SaPFactory_EpetraVsTpetra
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
using Teuchos::ParameterList;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100, nz = 100;
Galeri::Xpetra::Parameters<GO> galeriParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file [default = 'scalingTest.xml']");
bool printTimings = true; clp.setOption("timings", "notimings", &printTimings, "print timings to screen");
int writeMatricesOPT = -2; clp.setOption("write", &writeMatricesOPT, "write matrices to file (-1 means all; i>=0 means level i)");
std::string dsolveType = "cg", solveType; clp.setOption("solver", &dsolveType, "solve type: (none | cg | gmres | standalone)");
double dtol = 1e-12, tol; clp.setOption("tol", &dtol, "solver convergence tolerance");
std::string mapFile; clp.setOption("map", &mapFile, "map data file");
std::string matrixFile; clp.setOption("matrix", &matrixFile, "matrix data file");
std::string coordFile; clp.setOption("coords", &coordFile, "coordinates data file");
int numRebuilds = 0; clp.setOption("rebuild", &numRebuilds, "#times to rebuild hierarchy");
int maxIts = 200; clp.setOption("its", &maxIts, "maximum number of solver iterations");
bool scaleResidualHistory = true; clp.setOption("scale", "noscale", &scaleResidualHistory, "scaled Krylov residual history");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<ParameterList>(¶mList), *comm);
bool isDriver = paramList.isSublist("Run1");
if (isDriver) {
// update galeriParameters with the values from the XML file
ParameterList& realParams = galeriParameters.GetParameterList();
for (ParameterList::ConstIterator it = realParams.begin(); it != realParams.end(); it++) {
const std::string& name = realParams.name(it);
if (paramList.isParameter(name))
realParams.setEntry(name, paramList.getEntry(name));
}
}
// Retrieve matrix parameters (they may have been changed on the command line)
// [for instance, if we changed matrix type from 2D to 3D we need to update nz]
ParameterList galeriList = galeriParameters.GetParameterList();
// =========================================================================
// Problem construction
// =========================================================================
std::ostringstream galeriStream;
comm->barrier();
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
RCP<TimeMonitor> tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<Matrix> A;
RCP<const Map> map;
RCP<MultiVector> coordinates;
RCP<MultiVector> nullspace;
if (matrixFile.empty()) {
galeriStream << "========================================================\n" << xpetraParameters << galeriParameters;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
std::string matrixType = galeriParameters.GetMatrixType();
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriList);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriList);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriList);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
galeriStream << "Processor subdomains in x direction: " << galeriList.get<int>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriList.get<int>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriList.get<int>("mz") << std::endl
<< "========================================================" << std::endl;
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") {
// Our default test case for elasticity: all boundaries of a square/cube have Neumann b.c. except left which has Dirichlet
galeriList.set("right boundary" , "Neumann");
galeriList.set("bottom boundary", "Neumann");
galeriList.set("top boundary" , "Neumann");
galeriList.set("front boundary" , "Neumann");
galeriList.set("back boundary" , "Neumann");
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(galeriParameters.GetMatrixType(), map, galeriList);
A = Pr->BuildMatrix();
nullspace = MultiVectorFactory::Build(map, 1);
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((galeriParameters.GetMatrixType() == "Elasticity2D") ? 2 : 3);
} else {
nullspace->putScalar(one);
}
} else {
if (!mapFile.empty())
map = Utils2::ReadMap(mapFile, xpetraParameters.GetLib(), comm);
comm->barrier();
if (lib == Xpetra::UseEpetra) {
A = Utils::Read(matrixFile, map);
} else {
// Tpetra matrix reader is still broken, so instead we read in
// a matrix in a binary format and then redistribute it
const bool binaryFormat = true;
A = Utils::Read(matrixFile, lib, comm, binaryFormat);
RCP<Matrix> newMatrix = MatrixFactory::Build(map, 1);
RCP<Import> importer = ImportFactory::Build(A->getRowMap(), map);
newMatrix->doImport(*A, *importer, Xpetra::INSERT);
newMatrix->fillComplete();
A.swap(newMatrix);
}
comm->barrier();
if (!coordFile.empty())
coordinates = Utils2::ReadMultiVector(coordFile, map);
nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(one);
}
comm->barrier();
tm = Teuchos::null;
galeriStream << "Galeri complete.\n========================================================" << std::endl;
int numReruns = 1;
if (paramList.isParameter("number of reruns"))
numReruns = paramList.get<int>("number of reruns");
const bool mustAlreadyExist = true;
for (int rerunCount = 1; rerunCount <= numReruns; rerunCount++) {
ParameterList mueluList, runList;
bool stop = false;
if (isDriver) {
runList = paramList.sublist("Run1", mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} else {
mueluList = paramList;
stop = true;
}
int runCount = 1;
do {
A->SetMaxEigenvalueEstimate(-one);
solveType = dsolveType;
tol = dtol;
int savedOut = -1;
FILE* openedOut = NULL;
if (isDriver) {
if (runList.isParameter("filename")) {
// Redirect all output into a filename We have to redirect all output,
// including printf's, therefore we cannot simply replace C++ cout
// buffers, and have to use heavy machinary (dup2)
std::string filename = runList.get<std::string>("filename");
if (numReruns > 1)
filename += "_run" + MueLu::toString(rerunCount);
filename += (lib == Xpetra::UseEpetra ? ".epetra" : ".tpetra");
savedOut = dup(STDOUT_FILENO);
openedOut = fopen(filename.c_str(), "w");
dup2(fileno(openedOut), STDOUT_FILENO);
}
if (runList.isParameter("solver")) solveType = runList.get<std::string>("solver");
if (runList.isParameter("tol")) tol = runList.get<double> ("tol");
}
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
fancyout << galeriStream.str();
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
RCP<HierarchyManager> mueLuFactory = rcp(new ParameterListInterpreter(mueluList));
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
RCP<Hierarchy> H;
for (int i = 0; i <= numRebuilds; i++) {
A->SetMaxEigenvalueEstimate(-one);
H = mueLuFactory->CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
if (!coordinates.is_null())
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory->SetupHierarchy(*H);
}
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
{
// we set seed for reproducibility
Utils::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
tm = Teuchos::null;
if (writeMatricesOPT > -2) {
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3.5 - Matrix output")));
H->Write(writeMatricesOPT, writeMatricesOPT);
tm = Teuchos::null;
}
comm->barrier();
if (solveType == "none") {
// Do not perform a solve
} else if (solveType == "standalone") {
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Fixed Point Solve")));
H->IsPreconditioner(false);
H->Iterate(*B, *X, maxIts);
} else if (solveType == "cg" || solveType == "gmres") {
#ifdef HAVE_MUELU_BELOS
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp <SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
fancyout << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
if (!scaleResidualHistory)
belosList.set("Implicit Residual Scaling", "None");
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg") {
solver = rcp(new Belos::PseudoBlockCGSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
} else if (solveType == "gmres") {
solver = rcp(new Belos::BlockGmresSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
}
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
fancyout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
} catch(...) {
fancyout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged)
fancyout << std::endl << "ERROR: Belos did not converge! " << std::endl;
else
fancyout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif //ifdef HAVE_MUELU_BELOS
} else {
throw MueLu::Exceptions::RuntimeError("Unknown solver type: \"" + solveType + "\"");
}
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union);
TimeMonitor::clearCounters();
if (isDriver) {
if (openedOut != NULL) {
dup2(savedOut, STDOUT_FILENO);
fclose(openedOut);
openedOut = NULL;
}
try {
runList = paramList.sublist("Run" + MueLu::toString(++runCount), mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} catch (std::exception) {
stop = true;
}
}
} while (stop == false);
}
return 0;
} //main
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
typedef Tpetra::Vector<SC,LO,GO,NO> TVEC;
typedef Tpetra::MultiVector<SC,LO,GO,NO> TMV;
typedef Tpetra::CrsMatrix<SC,LO,GO,NO,LMO> TCRS;
typedef Xpetra::CrsMatrix<SC,LO,GO,NO,LMO> XCRS;
typedef Xpetra::TpetraCrsMatrix<SC,LO,GO,NO,LMO> XTCRS;
typedef Xpetra::Matrix<SC,LO,GO,NO,LMO> XMAT;
typedef Xpetra::CrsMatrixWrap<SC,LO,GO,NO,LMO> XWRAP;
typedef Belos::OperatorT<TMV> TOP;
typedef Belos::OperatorTraits<SC,TMV,TOP> TOPT;
typedef Belos::MultiVecTraits<SC,TMV> TMVT;
typedef Belos::LinearProblem<SC,TMV,TOP> TProblem;
typedef Belos::SolverManager<SC,TMV,TOP> TBelosSolver;
typedef Belos::BlockGmresSolMgr<SC,TMV,TOP> TBelosGMRES;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
Teuchos::CommandLineProcessor clp(false);
GO nx,ny,nz;
nx=100; ny=100; nz=100;
double stretchx, stretchy, stretchz, h, delta;
stretchx=1.0; stretchy=1.0; stretchz=1.0;
h=0.01; delta=2.0;
int PMLXL, PMLXR, PMLYL, PMLYR, PMLZL, PMLZR;
PMLXL=10; PMLXR=10; PMLYL=10; PMLYR=10; PMLZL=10; PMLZR=10;
double omega, shift;
omega=20.0*M_PI;
shift=0.5;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Helmholtz1D", 0, stretchx, stretchy, stretchz,
h, delta, PMLXL, PMLXR, PMLYL, PMLYR, PMLZL, PMLZR, omega, shift);
Xpetra::Parameters xpetraParameters(clp);
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
RCP<TimeMonitor> tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
RCP<MultiVector> coordinates;
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx", nx));
galeriList.set("ny", pl.get("ny", ny));
galeriList.set("nz", pl.get("nz", nz));
RCP<const Map> map;
if (matrixParameters.GetMatrixType() == "Helmholtz1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Helmholtz2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Helmholtz3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
RCP<const Tpetra::Map<LO, GO, NO> > tmap = Xpetra::toTpetra(map);
Teuchos::ParameterList matrixParams = matrixParameters.GetParameterList();
// Build problem
RCP<Galeri::Xpetra::Problem_Helmholtz<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem_Helmholtz<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParams);
RCP<Matrix> A = Pr->BuildMatrix();
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map,1);
nullspace->putScalar( (SC) 1.0);
comm->barrier();
tm = Teuchos::null;
// Construct a multigrid preconditioner
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
// Multigrid Hierarchy
RCP<Hierarchy> H = rcp(new Hierarchy(A));
H->GetLevel(0)->Set("Nullspace",nullspace);
FactoryManager Manager;
H->Setup(Manager, 0, 5);
//H->Write(-1,-1);
tm = Teuchos::null;
// Solve Ax = b
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<TVEC> X = Tpetra::createVector<SC,LO,GO,NO>(tmap);
RCP<TVEC> B = Tpetra::createVector<SC,LO,GO,NO>(tmap);
X->putScalar((SC) 0.0);
B->putScalar((SC) 0.0);
if(comm->getRank()==0) {
B->replaceGlobalValue(0, (SC) 1.0);
}
tm = Teuchos::null;
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Belos Solve")));
// Define Operator and Preconditioner
RCP<TOP> belosOp = rcp(new Belos::XpetraOp<SC,LO,GO,NO,LMO> (A) ); // Turns a Xpetra::Matrix object into a Belos operator
RCP<TOP> belosPrec = rcp(new Belos::MueLuOp<SC,LO,GO,NO,LMO> (H) ); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP<TProblem> belosProblem = rcp(new TProblem(belosOp,X,B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
if(comm->getRank()==0)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 100;
double tol = 1e-6;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Flexible Gmres", false); // set flexible GMRES on/off
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency",1);
belosList.set("Output Style",Belos::Brief);
// Create solver manager
RCP<TBelosSolver> solver = rcp( new TBelosGMRES(belosProblem, rcp(&belosList, false)) );
// Perform solve
Belos::ReturnType ret=Belos::Unconverged;
try {
ret = solver->solve();
if (comm->getRank() == 0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
}
catch(...) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged) {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
} else {
if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
}
// Get the number of iterations for this solve.
if(comm->getRank()==0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
TimeMonitor::summarize();
} //main
int main(int argc, char *argv[]) {
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false); // Note:
GO nx,ny,nz;
nx=500;
ny=500;
nz=100;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'scalingTest.xml'");
int amgAsPrecond=1; clp.setOption("precond",&amgAsPrecond,"apply multigrid as preconditioner");
int amgAsSolver=0; clp.setOption("fixPoint",&amgAsSolver,"apply multigrid as solver");
bool printTimings=true; clp.setOption("timings","notimings",&printTimings,"print timings to screen");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) {
std::cout << "========================================================" << std::endl
<< xpetraParameters << matrixParameters;
}
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
// read aggregation options from file
Teuchos::FileInputSource fileSrc(xmlFileName);
Teuchos::XMLObject fileXML = fileSrc.getObject();
Teuchos::XMLParameterListReader listReader;
Teuchos::ParameterList aggList = listReader.toParameterList(fileXML);
//std::cout << "===========aggList start===========" << std::endl;
//aggList.print(std::cout);
//std::cout << "===========aggList end===========" << std::endl;
// instantiate aggregate factory, set options from parameter list
RCP<MueLu::SingleLevelFactoryBase> aggFact;
if (aggList.name() == "UncoupledAggregationFactory") {
RCP<UncoupledAggregationFactory> ucFact = rcp( new UncoupledAggregationFactory() );
//ucFact->SetParameterList(aggList);
//FIXME hack until UCAgg uses PL interface
std::string ordering = aggList.get<std::string>("Ordering");
MueLu::AggOptions::Ordering eordering;
if (ordering=="Natural") eordering = MueLu::AggOptions::NATURAL;
if (ordering=="Graph") eordering = MueLu::AggOptions::GRAPH;
if (ordering=="Random") eordering = MueLu::AggOptions::RANDOM;
ucFact->SetOrdering(eordering);
ucFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
ucFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = ucFact;
} else if (aggList.name() == "CoupledAggregationFactory") {
RCP<CoupledAggregationFactory> cFact = rcp( new CoupledAggregationFactory() );
//cFact->SetParameterList(aggList);
//FIXME hack until CoupledAgg uses PL interface
//cFact->SetOrdering(aggList.get<std::string>("Ordering"));
cFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
cFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = cFact;
} else {
throw(MueLu::Exceptions::RuntimeError("List's name does not correspond to a known aggregation factory."));
}
//Teuchos::ParameterList tlist = aggFact->GetParameterList();
//std::cout << "===========verify List start===========" << std::endl;
//tlist.print(std::cout);
//std::cout << "===========verify List end===========" << std::endl;
// build matrix
RCP<TimeMonitor> tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<const Map> map;
RCP<MultiVector> coordinates;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx",nx));
galeriList.set("ny", pl.get("ny",ny));
//galeriList.set("mx", comm->getSize());
//galeriList.set("my", 1);
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D",map,matrixParameters.GetParameterList());
//map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList); //TODO when available in Galeri
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
}
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
std::cout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "========================================================" << std::endl;
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
tm = Teuchos::null;
Level level;
RCP<MueLu::FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
level.SetLevelID(0);
level.Set("A", A);
level.Request("Aggregates", aggFact.get());
level.Request(*aggFact);
level.setVerbLevel(Teuchos::VERB_NONE);
aggFact->setVerbLevel(Teuchos::VERB_NONE);
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("aggregation time")));
aggFact->Build(level);
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize();
} //main
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
bool success = false;
bool verbose = true;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int numProc = comm->getSize();
int myRank = comm->getRank();
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& out = *fancy;
out.setOutputToRootOnly(0);
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
Xpetra::Parameters xpetraParameters(clp);
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
const int numLists = 1;
std::vector<std::string> dirList;
dirList.push_back("Convergence/Laplace2D/");
bool failed = false;
for (int k = 0; k < numLists; k++) {
const std::string& dirName = dirList[k];
std::string problemFile = dirName + "problem.xml";
ParameterList galeriParameters;
Teuchos::updateParametersFromXmlFileAndBroadcast(problemFile, Teuchos::Ptr<Teuchos::ParameterList>(&galeriParameters), *comm);
if (!galeriParameters.isParameter("mz"))
galeriParameters.set<int>("mz", -1);
// =========================================================================
// Problem construction (copy-paste from Driver.cpp)
// =========================================================================
RCP<Matrix> A;
RCP<Map> map;
RCP<MultiVector> nullspace, coordinates;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriParameters.
std::string matrixType = galeriParameters.get<std::string>("matrixType");
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian1D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriParameters);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian2D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriParameters);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian3D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriParameters);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
#if 0
out << "========================================================\n" << xpetraParameters << galeriParameters;
out << "Processor subdomains in x direction: " << galeriParameters.get<GO>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriParameters.get<GO>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriParameters.get<GO>("mz") << std::endl
<< "========================================================" << std::endl;
#endif
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixType, map, galeriParameters);
A = Pr->BuildMatrix();
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((matrixType == "Elasticity2D") ? 2 : 3);
} else {
nullspace = MultiVectorFactory::Build(map, 1);
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(0);
for (int i = 0; i < nsData.size(); i++)
nsData[i] = one;
}
// =========================================================================
// Run different configurations
// =========================================================================
Teuchos::ArrayRCP<std::string> fileList = MueLuTests::TestHelpers::GetFileList(dirList[k],
(numProc == 1 ? std::string(".xml") : std::string("_np" + Teuchos::toString(numProc) + ".xml")));
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
for (int i = 0; i < fileList.size(); i++) {
if (fileList[i] == "problem.xml")
continue;
// Set seed
Utilities::SetRandomSeed(*comm);
// Reset (potentially) cached value of the estimate
A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());
std::string xmlFile = dirName + fileList[i];
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFile, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
std::string solveType = paramList.get<std::string> ("solver", "standalone");
double goldRate = paramList.get<double> ("convergence rate");
ParameterList& mueluList = paramList.sublist ("MueLu");
TEUCHOS_TEST_FOR_EXCEPTION(solveType != "standalone" && solveType != "cg" && solveType != "gmres", MueLu::Exceptions::RuntimeError,
"Unknown solver type \"" << solveType << "\"");
bool isPrec = !(solveType == "standalone");
if (!mueluList.isParameter("verbosity"))
mueluList.set("verbosity", "none");
#ifndef HAVE_MUELU_BELOS
if (isPrec)
out << xmlFile << ": skipped (Belos is not enabled)" << std::endl;
#endif
// =========================================================================
// Preconditioner construction
// =========================================================================
RCP<Hierarchy> H;
try {
ParameterListInterpreter mueluFactory(mueluList);
H = mueluFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("Coordinates", coordinates);
mueluFactory.SetupHierarchy(*H);
} catch (Teuchos::ExceptionBase& e) {
std::string msg = e.what();
msg = msg.substr(msg.find_last_of('\n')+1);
out << "Caught exception: " << msg << std::endl;
if (msg == "Zoltan interface is not available" ||
msg == "Zoltan2 interface is not available") {
if (myRank == 0)
out << xmlFile << ": skipped (missing library)" << std::endl;
continue;
}
}
// Set X, B
{
// TODO: do multiple vectors simultaneously to average
// we set seed for reproducibility
Utilities::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
const int maxIts = 100;
const double tol = 1e-12;
H->IsPreconditioner(isPrec);
if (isPrec == false) {
MueLu::ReturnType ret = H->Iterate(*B, *X, std::pair<LO,SC>(maxIts, tol));
double rate = H->GetRate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} else {
#ifdef HAVE_MUELU_BELOS
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
// Define Operator and Preconditioner
RCP<OP> belosOp = rcp(new Belos::XpetraOp<SC, LO, GO, NO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
RCP<OP> belosPrec = rcp(new Belos::MueLuOp <SC, LO, GO, NO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
out << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
#if 1
belosList.set("Verbosity", Belos::Errors + Belos::Warnings);
#else
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
#endif
// Belos custom test to store residuals
// Create an iterative solver manager
RCP<Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg")
solver = rcp(new Belos::PseudoBlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
else if (solveType == "gmres")
solver = rcp(new Belos::BlockGmresSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
RCP<Belos::MyStatusTest<SC, MV, OP> > status = rcp(new Belos::MyStatusTest<SC, MV, OP>(tol));
solver->setDebugStatusTest(status);
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
double rate = status->rate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} catch(...) {
out << xmlFile << ": failed (exception)" << std::endl;
failed = true;
}
#endif //ifdef HAVE_MUELU_BELOS
}
}
}
success = !failed;
out << std::endl << "End Result: TEST " << (failed ? "FAILED" : "PASSED") << std::endl;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main_(Teuchos::CommandLineProcessor &clp, Xpetra::UnderlyingLib lib, int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
//
// MPI initialization
//
bool success = false;
bool verbose = true;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Galeri::Xpetra::Parameters<GlobalOrdinal> matrixParameters(clp, 256); // manage parameters of the test case
std::string xmlFileName = "stratimikos_ParameterList.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'stratimikos_ParameterList.xml'.");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
// Read in parameter list
Teuchos::RCP<Teuchos::ParameterList> paramList = Teuchos::getParametersFromXmlFile(xmlFileName);
//
// Construct the problem
//
RCP<const Map> map = MapFactory::createUniformContigMap(lib, matrixParameters.GetNumGlobalElements(), comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<CrsMatrixWrap> A = Pr->BuildMatrix();
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
{
// we set seed for reproducibility
Utilities::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, Teuchos::ScalarTraits<Scalar>::one(), Teuchos::ScalarTraits<Scalar>::zero());
Teuchos::Array<typename Teuchos::ScalarTraits<Scalar>::magnitudeType> norms(1);
B->norm2(norms);
B->scale(Teuchos::ScalarTraits<Scalar>::one()/norms[0]);
X->putScalar(Teuchos::ScalarTraits<Scalar>::zero());
}
//
// Build Thyra linear algebra objects
//
RCP<const Thyra::LinearOpBase<Scalar> > thyraA = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyra(A->getCrsMatrix());
RCP< Thyra::VectorBase<Scalar> >thyraX = Teuchos::rcp_const_cast<Thyra::VectorBase<Scalar> >(Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyraVector(X));
RCP<const Thyra::VectorBase<Scalar> >thyraB = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyraVector(B);
//
// Build Stratimikos solver
//
Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder; // This is the Stratimikos main class (= factory of solver factory).
Stratimikos::enableMueLu<LocalOrdinal,GlobalOrdinal,Node>(linearSolverBuilder); // Register MueLu as a Stratimikos preconditioner strategy.
linearSolverBuilder.setParameterList(paramList); // Setup solver parameters using a Stratimikos parameter list.
// Build a new "solver factory" according to the previously specified parameter list.
RCP<Thyra::LinearOpWithSolveFactoryBase<Scalar> > solverFactory = Thyra::createLinearSolveStrategy(linearSolverBuilder);
// Build a Thyra operator corresponding to A^{-1} computed using the Stratimikos solver.
Teuchos::RCP<Thyra::LinearOpWithSolveBase<Scalar> > thyraInverseA = Thyra::linearOpWithSolve(*solverFactory, thyraA);
//
// Solve Ax = b.
//
Thyra::SolveStatus<Scalar> status = Thyra::solve<Scalar>(*thyraInverseA, Thyra::NOTRANS, *thyraB, thyraX.ptr());
std::cout << status << std::endl;
success = (status.solveStatus == Thyra::SOLVE_STATUS_CONVERGED);
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
/**********************************************************************************/
/* SET TEST PARAMETERS */
/**********************************************************************************/
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561=3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748=4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// custom parameters
int nSmoothers=2;
LO maxLevels = 3;
LO its=10;
std::string coarseSolver="ifpack2";
// std::string coarseSolver="amesos2";
int pauseForDebugger=0;
clp.setOption("nSmoothers",&nSmoothers,"number of Gauss-Seidel smoothers in the MergedSmootehrs");
clp.setOption("maxLevels",&maxLevels,"maximum number of levels allowed. If 1, then a MergedSmoother is used on the coarse grid");
clp.setOption("its",&its,"number of multigrid cycles");
clp.setOption("coarseSolver",&coarseSolver,"amesos2 or ifpack2 (Tpetra specific. Ignored for Epetra)");
clp.setOption("debug",&pauseForDebugger,"pause to attach debugger");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
if (comm->getRank() == 0) {
// matrixParameters.print();
// xpetraParameters.print();
// TODO: print custom parameters
}
if (pauseForDebugger) {
Utils::PauseForDebugger();
}
/**********************************************************************************/
/* CREATE INITIAL MATRIX */
/**********************************************************************************/
const RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
RCP<Matrix> Op = Pr->BuildMatrix();
#ifdef NEUMANN
// Tranform matrix to Neumann b.c.
// Essentially, we need to update two diagonal elements
// TODO: calls to getLocalRowView not really needed
Op->resumeFill();
Teuchos::ArrayView<const LO> indices;
Teuchos::ArrayView<const SC> values;
Teuchos::Array<SC> newValues(2, 0.0);
size_t myRank = Op->getRowMap()->getComm()->getRank();
size_t nCpus = Op->getRowMap()->getComm()->getSize();
if (myRank == 0) { // JG TODO: can we use rowMap->isNodeLocalElement(0) instead for more genericity?
//LO firstRow = 0;
newValues[0] = 1.0; newValues[1] = -1.0;
Op->getLocalRowView(0, indices, values);
Op->replaceLocalValues(0, indices, newValues);
}
if (myRank == nCpus-1) { // JG TODO: can we use rowMap->isNodeLocalElement(lastRow) instead for more genericity?
LO lastRow = Op->getNodeNumRows()-1;
newValues[0] = -1.0; newValues[1] = 1.0;
Op->getLocalRowView(lastRow, indices, values);
Op->replaceLocalValues(lastRow, indices, newValues);
}
Op->fillComplete();
#endif // NEUMANN
/**********************************************************************************/
/* */
/**********************************************************************************/
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
std::cout << "||NS|| = " << norms[0] << std::endl;
RCP<MueLu::Hierarchy<SC,LO,GO,NO,LMO> > H = rcp( new Hierarchy() );
H->SetDefaultVerbLevel(MueLu::Extreme);
H->IsPreconditioner(false);
RCP<MueLu::Level> Finest = H->GetLevel();
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A", Op);
Finest->Set("Nullspace", nullSpace);
FactoryManager M;
M.SetFactory("Aggregates", rcp(new CoupledAggregationFactory()));
M.SetFactory("Ptent", rcp(new TentativePFactory()));
M.SetFactory("P", rcp(new SaPFactory()));
#ifdef EMIN
// Energy-minimization
RCP<PatternFactory> PatternFact = rcp(new PatternFactory());
#if 0
PatternFact->SetFactory("P", M.GetFactory("Ptent"));
#else
PatternFact->SetFactory("P", M.GetFactory("P"));
#endif
M.SetFactory("Ppattern", PatternFact);
RCP<EminPFactory> EminPFact = rcp(new EminPFactory());
EminPFact->SetFactory("P", M.GetFactory("Ptent"));
M.SetFactory("P", EminPFact);
RCP<NullspacePresmoothFactory> NullPreFact = rcp(new NullspacePresmoothFactory());
NullPreFact->SetFactory("Nullspace", M.GetFactory("Nullspace"));
M.SetFactory("Nullspace", NullPreFact);
#endif
RCP<SmootherPrototype> smooProto = gimmeMergedSmoother(nSmoothers, xpetraParameters.GetLib(), coarseSolver, comm->getRank());
M.SetFactory("Smoother", rcp(new SmootherFactory(smooProto)));
Teuchos::ParameterList status;
RCP<SmootherPrototype> coarseProto;
if (maxLevels != 1)
coarseProto = gimmeCoarseProto(xpetraParameters.GetLib(), coarseSolver, comm->getRank());
else
coarseProto = gimmeMergedSmoother(nSmoothers, xpetraParameters.GetLib(), coarseSolver, comm->getRank());
if (coarseProto == Teuchos::null)
return EXIT_FAILURE;
#ifdef NEUMANN
// Use coarse level projection solver
RCP<SmootherPrototype> projectedSolver = rcp(new ProjectorSmoother(coarseProto));
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(projectedSolver));
#else
RCP<SmootherFactory> coarseSolveFact = rcp(new SmootherFactory(coarseProto));
#endif
M.SetFactory("CoarseSolver", coarseSolveFact);
H->EnableGraphDumping("graph.dot", 2);
H->Setup(M, 0, maxLevels);
//if (comm->getRank() == 0) {
// std::cout << "======================\n Multigrid statistics \n======================" << std::endl;
// status.print(std::cout,Teuchos::ParameterList::PrintOptions().indent(2));
//}
// Define RHS
RCP<MultiVector> X = MultiVectorFactory::Build(map,1);
RCP<MultiVector> RHS = MultiVectorFactory::Build(map,1);
X->setSeed(846930886);
X->randomize();
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_true|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
Op->apply(*X,*RHS,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
// Use AMG directly as an iterative method
{
X->putScalar( (SC) 0.0);
H->Iterate(*RHS,its,*X);
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_" << std::setprecision(2) << its << "|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
}
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
using Teuchos::RCP; // reference count pointers
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false); // Note:
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 5000); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
std::string xmlFileName = "muelu_ParameterList.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'muelu_ParameterList.xml'");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) { std::cout << xpetraParameters << matrixParameters; }
//
// Construct the problem
//
RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Map, CrsMatrixWrap, MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
//
// Construct a multigrid preconditioner
//
// Multigrid Hierarchy
ParameterListInterpreter mueLuFactory(xmlFileName,*comm);
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
mueLuFactory.SetupHierarchy(*H);
//
// Solve Ax = b
//
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
X->putScalar((Scalar) 0.0);
B->setSeed(846930886); B->randomize();
// Use AMG directly as an iterative solver (not as a preconditionner)
int nIts = 9;
H->Iterate(*B, nIts, *X);
// Print relative residual norm
ST::magnitudeType residualNorms = Utils::ResidualNorm(*A, *X, *B)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// =========================================================================
// Convenient definitions
// =========================================================================
SC zero = Teuchos::ScalarTraits<SC>::zero(), one = Teuchos::ScalarTraits<SC>::one();
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100, nz = 100;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace1D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "tutorial1a.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'tutorial1a.xml'");
int amgAsPrecond = 1; clp.setOption("precond", &amgAsPrecond, "apply multigrid as preconditioner");
int amgAsSolver = 0; clp.setOption("fixPoint", &amgAsSolver, "apply multigrid as solver");
bool printTimings = true; clp.setOption("timings", "notimings", &printTimings, "print timings to screen");
int writeMatricesOPT = -2; clp.setOption("write", &writeMatricesOPT, "write matrices to file (-1 means all; i>=0 means level i)");
double tol = 1e-12; clp.setOption("tol", &tol, "solver convergence tolerance");
std::string krylovMethod = "cg"; clp.setOption("krylov", &krylovMethod, "outer Krylov method");
std::string mapFile; clp.setOption("map", &mapFile, "map data file");
std::string matrixFile; clp.setOption("matrix", &matrixFile, "matrix data file");
std::string coordFile; clp.setOption("coords", &coordFile, "coordinates data file");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
// =========================================================================
// Problem construction
// =========================================================================
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time"))), tm;
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<Matrix> A;
RCP<MultiVector> coordinates;
RCP<MultiVector> nullspace;
if (matrixFile.empty()) {
fancyout << "========================================================\n" << xpetraParameters << matrixParameters;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx",nx));
galeriList.set("ny", pl.get("ny",ny));
galeriList.set("nz", pl.get("nz",nz));
// galeriList.set("mx", comm->getSize());
// galeriList.set("my", 1);
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
RCP<const Map> map;
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D" || matrixParameters.GetMatrixType() == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D" || matrixParameters.GetMatrixType() == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D",map,matrixParameters.GetParameterList());
}
// Expand map to do multiple DOF per node for block problems
if (matrixParameters.GetMatrixType() == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixParameters.GetMatrixType() == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
GO mz = galeriList.get("mz", -1);
fancyout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "Processor subdomains in z direction: " << mz << std::endl
<< "========================================================" << std::endl;
}
Teuchos::ParameterList matrixParams = matrixParameters.GetParameterList();
matrixParams.set("mx", galeriList.get("mx", -1));
matrixParams.set("my", galeriList.get("my", -1));
matrixParams.set("mz", galeriList.get("mz", -1));
if (matrixParameters.GetMatrixType() == "Elasticity2D" || matrixParameters.GetMatrixType() == "Elasticity3D") {
// Our default test case for elasticity: all boundaries of a square/cube have Neumann b.c. except left which has Dirichlet
matrixParams.set("right boundary" , "Neumann");
matrixParams.set("bottom boundary", "Neumann");
matrixParams.set("top boundary" , "Neumann");
matrixParams.set("front boundary" , "Neumann");
matrixParams.set("back boundary" , "Neumann");
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParams);
A = Pr->BuildMatrix();
nullspace = MultiVectorFactory::Build(map, 1);
if (matrixParameters.GetMatrixType() == "Elasticity2D" ||
matrixParameters.GetMatrixType() == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((matrixParameters.GetMatrixType() == "Elasticity2D") ? 2 : 3);
} else {
nullspace->putScalar(one);
}
} else {
RCP<const Map> map = (mapFile.empty() ? Teuchos::null : Utils2::ReadMap(mapFile, xpetraParameters.GetLib(), comm));
comm->barrier();
A = Utils::Read(matrixFile, map);
comm->barrier();
coordinates = Utils2::ReadMultiVector(coordFile, map);
nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(one);
}
RCP<const Map> map = A->getRowMap();
comm->barrier();
tm = Teuchos::null;
fancyout << "Galeri complete.\n========================================================" << std::endl;
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
ParameterListInterpreter mueLuFactory(xmlFileName, *comm);
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory.SetupHierarchy(*H);
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
comm->barrier();
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
{
// we set seed for reproducibility
X->setSeed(846930886);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<Teuchos::ScalarTraits<double>::magnitudeType> norms(1);
B->norm2(norms);
B->scale(1.0/norms[0]);
X->putScalar(zero);
}
tm = Teuchos::null;
if (writeMatricesOPT > -2)
H->Write(writeMatricesOPT, writeMatricesOPT);
comm->barrier();
if (amgAsSolver) {
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Fixed Point Solve")));
H->IsPreconditioner(false);
H->Iterate(*B, 25, *X);
} else if (amgAsPrecond) {
#ifdef HAVE_MUELU_BELOS
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
fancyout << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 2000;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver;
if (krylovMethod == "cg") {
solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
} else if (krylovMethod == "gmres") {
solver = rcp(new Belos::BlockGmresSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
} else {
TEUCHOS_TEST_FOR_EXCEPTION(true, MueLu::Exceptions::RuntimeError, "Invalid Krylov method. Options are \"cg\" or \" gmres\".");
}
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
fancyout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
} catch(...) {
fancyout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged)
fancyout << std::endl << "ERROR: Belos did not converge! " << std::endl;
else
fancyout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif //ifdef HAVE_MUELU_BELOS
}
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings) {
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union);
//MueLu::MutuallyExclusiveTime<MueLu::BaseClass>::PrintParentChildPairs();
}
return 0;
} //main
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
/**********************************************************************************/
/* SET TEST PARAMETERS */
/**********************************************************************************/
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561=3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748=4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// custom parameters
LO maxLevels = 1;
LO its=2;
std::string coarseSolver="amesos2";
clp.setOption("maxLevels",&maxLevels,"maximum number of levels allowed");
clp.setOption("its",&its,"number of multigrid cycles");
clp.setOption("coarseSolver",&coarseSolver,"amesos2 or ifpack2 (Tpetra specific. Ignored for Epetra)");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
if (comm->getRank() == 0) {
matrixParameters.print();
xpetraParameters.print();
// TODO: print custom parameters
}
#ifdef FOR_PARALLEL_DEBUGGING
//Utils::BreakForDebugger(*comm);
LO mypid = comm->getRank();
if (mypid == 0) std::cout << "Host and Process Ids for tasks" << std::endl;
for (LO i = 0; i <comm->getSize(); i++) {
if (i == mypid ) {
char buf[80];
char hostname[80];
gethostname(hostname, sizeof(hostname));
LO pid = getpid();
sprintf(buf, "Host: %s\tMPI rank: %d,\tPID: %d\n\tattach %d\n\tcontinue\n",
hostname, mypid, pid, pid);
printf("%s\n",buf);
fflush(stdout);
sleep(1);
}
}
if (mypid == 0) {
printf( "** Enter a character to continue > "); fflush(stdout);
char go = ' ';
scanf("%c",&go);
}
comm->barrier();
#endif
/**********************************************************************************/
/* CREATE INITIAL MATRIX */
/**********************************************************************************/
const RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Map, CrsMatrixWrap, MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
RCP<Matrix> Op = Pr->BuildMatrix();
/**********************************************************************************/
/* */
/**********************************************************************************/
// dump matrix to file
//std::string fileName = "Amat.mm";
//Utils::Write(fileName,Op);
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(map,1);
nullSpace->putScalar( (SC) 1.0);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
nullSpace->norm1(norms);
if (comm->getRank() == 0)
std::cout << "||NS|| = " << norms[0] << std::endl;
RCP<MueLu::Hierarchy<SC,LO,GO,NO,LMO> > H = rcp( new Hierarchy() );
H->setDefaultVerbLevel(Teuchos::VERB_HIGH);
RCP<MueLu::Level> Finest = rcp( new MueLu::Level() );
Finest->setDefaultVerbLevel(Teuchos::VERB_HIGH);
Finest->Set("A",Op);
Finest->Set("Nullspace",nullSpace);
Finest->Request("Nullspace"); //FIXME putting this in to avoid error until Merge needs business
//FIXME is implemented
Finest->Set("NullSpace",nullSpace);
H->SetLevel(Finest);
RCP<CoupledAggregationFactory> CoupledAggFact = rcp(new CoupledAggregationFactory());
CoupledAggFact->SetMinNodesPerAggregate(3);
CoupledAggFact->SetMaxNeighAlreadySelected(0);
CoupledAggFact->SetOrdering("natural");
CoupledAggFact->SetPhase3AggCreation(0.5);
RCP<TentativePFactory> TentPFact = rcp(new TentativePFactory(CoupledAggFact));
RCP<SaPFactory> Pfact = rcp( new SaPFactory(TentPFact) );
//Pfact->SetDampingFactor(0.);
RCP<Factory> Rfact = rcp( new TransPFactory() );
RCP<GenericPRFactory> PRfact = rcp( new GenericPRFactory(Pfact,Rfact));
RCP<RAPFactory> Acfact = rcp( new RAPFactory() );
RCP<SmootherPrototype> smooProto;
Teuchos::ParameterList ifpackList;
ifpackList.set("relaxation: sweeps", (LO) 1);
ifpackList.set("relaxation: damping factor", (SC) 1.0);
/*
ifpackList.set("type", "Chebyshev");
ifpackList.set("chebyshev: degree", (int) 1);
ifpackList.set("chebyshev: max eigenvalue", (double) 2.0);
ifpackList.set("chebyshev: min eigenvalue", (double) 1.0);
ifpackList.set("chebyshev: zero starting solution", false);
*/
if (xpetraParameters.GetLib() == Xpetra::UseEpetra) {
#if defined(HAVE_MUELU_EPETRA) && defined(HAVE_MUELU_IFPACK)
ifpackList.set("relaxation: type", "symmetric Gauss-Seidel");
smooProto = rcp( new IfpackSmoother("point relaxation stand-alone",ifpackList) );
#endif
} else if (xpetraParameters.GetLib() == Xpetra::UseTpetra) {
#if defined(HAVE_MUELU_TPETRA) && defined(HAVE_MUELU_IFPACK2)
ifpackList.set("relaxation: type", "Symmetric Gauss-Seidel");
smooProto = rcp( new Ifpack2Smoother("RELAXATION",ifpackList) );
#endif
}
if (smooProto == Teuchos::null) {
throw(MueLu::Exceptions::RuntimeError("main: smoother error"));
}
RCP<SmootherFactory> SmooFact = rcp( new SmootherFactory(smooProto) );
Acfact->setVerbLevel(Teuchos::VERB_HIGH);
Teuchos::ParameterList status;
status = H->FullPopulate(PRfact,Acfact,SmooFact,0,maxLevels);
//RCP<MueLu::Level> coarseLevel = H.GetLevel(1);
//RCP<Matrix> P = coarseLevel->template Get< RCP<Matrix> >("P");
//fileName = "Pfinal.mm";
//Utils::Write(fileName,P);
if (comm->getRank() == 0) {
std::cout << "======================\n Multigrid statistics \n======================" << std::endl;
status.print(std::cout,Teuchos::ParameterList::PrintOptions().indent(2));
}
//FIXME we should be able to just call smoother->SetNIts(50) ... but right now an exception gets thrown
RCP<SmootherPrototype> coarseProto;
if (xpetraParameters.GetLib() == Xpetra::UseEpetra) {
#if defined(HAVE_MUELU_EPETRA) && defined(HAVE_MUELU_AMESOS)
if (comm->getRank() == 0) std::cout << "CoarseGrid: AMESOS" << std::endl;
Teuchos::ParameterList amesosList;
amesosList.set("PrintTiming",true);
coarseProto = rcp( new AmesosSmoother("Amesos_Klu",amesosList) );
//#elif
#endif
} else if (xpetraParameters.GetLib() == Xpetra::UseTpetra) {
if (coarseSolver=="amesos2") {
#if defined(HAVE_MUELU_TPETRA) && defined(HAVE_MUELU_AMESOS2)
if (comm->getRank() == 0) std::cout << "CoarseGrid: AMESOS2" << std::endl;
Teuchos::ParameterList paramList; //unused
coarseProto = rcp( new Amesos2Smoother("Superlu", paramList) );
#else
std::cout << "AMESOS2 not available (try --coarseSolver=ifpack2)" << std::endl;
return EXIT_FAILURE;
#endif // HAVE_MUELU_TPETRA && HAVE_MUELU_AMESOS2
} else if(coarseSolver=="ifpack2") {
#if defined(HAVE_MUELU_TPETRA) && defined(HAVE_MUELU_IFPACK2)
if (comm->getRank() == 0) std::cout << "CoarseGrid: IFPACK2" << std::endl;
Teuchos::ParameterList ifpack2List;
ifpack2List.set("fact: ilut level-of-fill",99); // TODO ??
ifpack2List.set("fact: drop tolerance", 0);
ifpack2List.set("fact: absolute threshold", 0);
ifpack2List.set("fact: relative threshold", 0);
coarseProto = rcp( new Ifpack2Smoother("ILUT",ifpack2List) );
#else
std::cout << "IFPACK2 not available (try --coarseSolver=amesos2)" << std::endl;
return EXIT_FAILURE;
#endif
} else {
std::cout << "Unknow coarse grid solver (try --coarseSolver=ifpack2 or --coarseSolver=amesos2)" << std::endl;
return EXIT_FAILURE;
}
}
if (coarseProto == Teuchos::null) {
throw(MueLu::Exceptions::RuntimeError("main: coarse smoother error"));
}
SmootherFactory coarseSolveFact(coarseProto);
H->SetCoarsestSolver(coarseSolveFact,MueLu::PRE);
// Define RHS
RCP<MultiVector> X = MultiVectorFactory::Build(map,1);
RCP<MultiVector> RHS = MultiVectorFactory::Build(map,1);
X->setSeed(846930886);
X->randomize();
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_true|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
Op->apply(*X,*RHS,Teuchos::NO_TRANS,(SC)1.0,(SC)0.0);
// Use AMG directly as an iterative method
{
X->putScalar( (SC) 0.0);
H->PrintResidualHistory(true);
H->Iterate(*RHS,*X,its);
X->norm2(norms);
if (comm->getRank() == 0)
std::cout << "||X_" << std::setprecision(2) << its << "|| = " << std::setiosflags(std::ios::fixed) << std::setprecision(10) << norms[0] << std::endl;
}
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
//TODO: FIXME: option by default does not work for MueLu/Tpetra
int nIts = 9;
Teuchos::CommandLineProcessor clp(false); // Note:
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 256); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
std::string xmlFileName; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default an hard-coded parameter list.");
int muelu = true; clp.setOption("muelu", &muelu, "use muelu"); //TODO: bool instead of int
int ml = true;
#if defined(HAVE_MUELU_ML) && defined(HAVE_MUELU_EPETRA)
clp.setOption("ml", &ml, "use ml");
#endif
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
// TODO: check -ml and --linAlgebra
if (comm->getRank() == 0) { std::cout << xpetraParameters << matrixParameters; }
if (ml && xpetraParameters.GetLib() == Xpetra::UseTpetra) {
ml = false;
std::cout << "ML preconditionner can only be built if --linAlgebra=Epetra. Option --ml ignored" << std::endl;
}
//
// Construct the problem
//
RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC, LO, GO, Map, CrsMatrixWrap, MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
//
// Preconditionner configuration
//
// ML parameter list
RCP<Teuchos::ParameterList> params;
if (xmlFileName != "") {
std::cout << "Reading " << xmlFileName << " ..." << std::endl;
params = Teuchos::getParametersFromXmlFile(xmlFileName);
} else {
std::cout << "Using hard-coded parameter list:" << std::endl;
params = rcp(new Teuchos::ParameterList());
params->set("ML output", 10);
params->set("max levels", 2);
params->set("smoother: type", "symmetric Gauss-Seidel");
if (xpetraParameters.GetLib() == Xpetra::UseTpetra) // TODO: remove 'if' when Amesos2-KLU becomes available
params->set("coarse: type","Amesos-Superlu");
else
params->set("coarse: type","Amesos-KLU");
}
std::cout << "Initial parameter list" << std::endl;
std::cout << *params << std::endl;
if (muelu) {
//
// Construct a multigrid preconditioner
//
// Multigrid Hierarchy
MLParameterListInterpreter mueLuFactory(*params);
RCP<Hierarchy> H = mueLuFactory.CreateHierarchy();
// build default null space
LocalOrdinal numPDEs = 1;
if(A->IsView("stridedMaps")==true) {
Xpetra::viewLabel_t oldView = A->SwitchToView("stridedMaps"); // note: "stridedMaps are always non-overlapping (correspond to range and domain maps!)
numPDEs = Teuchos::rcp_dynamic_cast<const StridedMap>(A->getRowMap())->getFixedBlockSize();
oldView = A->SwitchToView(oldView);
}
RCP<MultiVector> nullspace = MultiVectorFactory::Build(A->getDomainMap(), numPDEs);
for (int i=0; i<numPDEs; ++i) {
Teuchos::ArrayRCP<Scalar> nsValues = nullspace->getDataNonConst(i);
int numBlocks = nsValues.size() / numPDEs;
for (int j=0; j< numBlocks; ++j) {
nsValues[j*numPDEs + i] = 1.0;
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("A", A);
//
// build hierarchy
//
mueLuFactory.SetupHierarchy(*H);
//
// Solve Ax = b
//
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
X->putScalar((Scalar) 0.0);
B->setSeed(846930886); B->randomize();
// AMG as a standalone solver
H->IsPreconditioner(false);
H->Iterate(*B, *X, nIts);
// Print relative residual norm
Teuchos::ScalarTraits<SC>::magnitudeType residualNorms = Utils::ResidualNorm(*A, *X, *B)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
#if defined(HAVE_MUELU_EPETRA) && defined(HAVE_MUELU_AZTECOO)
if (xpetraParameters.GetLib() == Xpetra::UseEpetra) { //TODO: should be doable with Tpetra too
// AMG as a preconditioner
//TODO: name mueluPrec and mlPrec not
H->IsPreconditioner(true);
MueLu::EpetraOperator mueluPrec(H); // Wrap MueLu preconditioner into an Epetra Operator
//
// Solve Ax = b
//
RCP<Epetra_CrsMatrix> eA; //duplicate code
{ // TODO: simplify this
RCP<CrsMatrixWrap> xCrsOp = Teuchos::rcp_dynamic_cast<CrsMatrixWrap>(A, true);
RCP<CrsMatrix> xCrsMtx = xCrsOp->getCrsMatrix();
RCP<EpetraCrsMatrix> eCrsMtx = Teuchos::rcp_dynamic_cast<EpetraCrsMatrix>(xCrsMtx, true);
eA = eCrsMtx->getEpetra_CrsMatrixNonConst();
}
RCP<Epetra_Vector> eX = rcp(new Epetra_Vector(eA->RowMap()));
RCP<Epetra_Vector> eB = rcp(new Epetra_Vector(eA->RowMap()));
eX->PutScalar((Scalar) 0.0);
eB->SetSeed(846930886); eB->Random();
Epetra_LinearProblem eProblem(eA.get(), eX.get(), eB.get());
// AMG as a standalone solver
AztecOO solver(eProblem);
solver.SetPrecOperator(&mueluPrec);
solver.SetAztecOption(AZ_solver, AZ_fixed_pt);
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(nIts, 1e-10);
{ //TODO: simplify this
RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(eX));
RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(eB));
// Print relative residual norm
Teuchos::ScalarTraits<SC>::magnitudeType residualNorms2 = Utils::ResidualNorm(*A, *mueluX, *mueluB)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms2 << std::endl;
}
// TODO: AMG as a preconditioner (AZ_cg)
}
#endif // HAVE_MUELU_AZTECOO
} // if (muelu)
#if defined(HAVE_MUELU_ML) && defined(HAVE_MUELU_EPETRA)
if (ml) {
std::cout << std::endl << std::endl << std::endl << std::endl << "**** ML ml ML ml ML" << std::endl << std::endl << std::endl << std::endl;
//
// Construct a multigrid preconditioner
//
// Multigrid Hierarchy
RCP<CrsMatrixWrap> crsOp = Teuchos::rcp_dynamic_cast<CrsMatrixWrap>(A, true);
RCP<CrsMatrix> crsMtx = crsOp->getCrsMatrix();
RCP<EpetraCrsMatrix> epetraCrsMtx = Teuchos::rcp_dynamic_cast<EpetraCrsMatrix>(crsMtx, true);
RCP<const Epetra_CrsMatrix> epetra_CrsMtx = epetraCrsMtx->getEpetra_CrsMatrix();
RCP<Epetra_CrsMatrix> eA;
{ // TODO: simplify this
RCP<CrsMatrixWrap> xCrsOp = Teuchos::rcp_dynamic_cast<CrsMatrixWrap>(A, true);
RCP<CrsMatrix> xCrsMtx = xCrsOp->getCrsMatrix();
RCP<EpetraCrsMatrix> eCrsMtx = Teuchos::rcp_dynamic_cast<EpetraCrsMatrix>(xCrsMtx, true);
eA = eCrsMtx->getEpetra_CrsMatrixNonConst();
}
RCP<ML_Epetra::MultiLevelPreconditioner> mlPrec = rcp(new ML_Epetra::MultiLevelPreconditioner(*eA, *params));
#ifdef HAVE_MUELU_AZTECOO
//
// Solve Ax = b
//
RCP<Epetra_Vector> eX = rcp(new Epetra_Vector(eA->RowMap()));
RCP<Epetra_Vector> eB = rcp(new Epetra_Vector(eA->RowMap()));
eX->PutScalar((Scalar) 0.0);
eB->SetSeed(846930886); eB->Random();
Epetra_LinearProblem eProblem(eA.get(), eX.get(), eB.get());
// AMG as a standalone solver
AztecOO solver(eProblem);
solver.SetPrecOperator(mlPrec.get());
solver.SetAztecOption(AZ_solver, AZ_fixed_pt);
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(nIts, 1e-10);
{ //TODO: simplify this
RCP<Vector> mueluX = rcp(new Xpetra::EpetraVector(eX));
RCP<Vector> mueluB = rcp(new Xpetra::EpetraVector(eB));
// Print relative residual norm
Teuchos::ScalarTraits<SC>::magnitudeType residualNorms = Utils::ResidualNorm(*A, *mueluX, *mueluB)[0];
if (comm->getRank() == 0)
std::cout << "||Residual|| = " << residualNorms << std::endl;
}
// TODO: AMG as a preconditioner (AZ_cg)
#else
std::cout << "Enable AztecOO to see solution" << std::endl;
#endif // HAVE_MUELU_AZTECOO
std::cout << "Parameter list after ML run" << std::endl;
const Teuchos::ParameterList & paramsAfterML = mlPrec->GetList();
std::cout << paramsAfterML << std::endl;
} // if (ml)
#endif // HAVE_MUELU_ML && HAVE_MUELU_EPETRA
return EXIT_SUCCESS;
}
void ConstructData(const std::string& matrixType, Teuchos::ParameterList& galeriList,
Xpetra::UnderlyingLib lib, Teuchos::RCP<const Teuchos::Comm<int> >& comm,
Teuchos::RCP<Xpetra::Matrix <Scalar,LocalOrdinal,GlobalOrdinal,Node> >& A,
Teuchos::RCP<const Xpetra::Map <LocalOrdinal,GlobalOrdinal, Node> >& map,
Teuchos::RCP<Xpetra::MultiVector <Scalar,LocalOrdinal,GlobalOrdinal,Node> >& coordinates,
Teuchos::RCP<Xpetra::MultiVector <Scalar,LocalOrdinal,GlobalOrdinal,Node> >& nullspace) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::ArrayRCP;
using Teuchos::RCP;
using Teuchos::TimeMonitor;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriList);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriList);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriList);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") {
// Our default test case for elasticity: all boundaries of a square/cube have Neumann b.c. except left which has Dirichlet
galeriList.set("right boundary" , "Neumann");
galeriList.set("bottom boundary", "Neumann");
galeriList.set("top boundary" , "Neumann");
galeriList.set("front boundary" , "Neumann");
galeriList.set("back boundary" , "Neumann");
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixType, map, galeriList);
A = Pr->BuildMatrix();
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((matrixType == "Elasticity2D") ? 2 : 3);
}
}
