Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::CrsMatrix <Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inA,
Teuchos::ParameterList& paramListIn,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> >& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
bool hasParamList = paramListIn.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = paramListIn;
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(inA);
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
Teuchos::RCP<MueLu::Hierarchy<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateXpetraPreconditioner(Teuchos::RCP<Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> > op,
const Teuchos::ParameterList& inParamList,
Teuchos::RCP<Xpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> > coords = Teuchos::null,
Teuchos::RCP<Xpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> > nullspace = Teuchos::null) {
typedef MueLu::HierarchyManager<Scalar,LocalOrdinal,GlobalOrdinal,Node> HierarchyManager;
typedef MueLu::HierarchyUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node> HierarchyUtils;
typedef MueLu::Hierarchy<Scalar,LocalOrdinal,GlobalOrdinal,Node> Hierarchy;
typedef MueLu::MLParameterListInterpreter<Scalar,LocalOrdinal,GlobalOrdinal,Node> MLParameterListInterpreter;
typedef MueLu::ParameterListInterpreter<Scalar,LocalOrdinal,GlobalOrdinal,Node> ParameterListInterpreter;
typedef Xpetra::MultiVectorFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node> MultiVectorFactory;
std::string timerName = "MueLu setup time";
RCP<Teuchos::Time> tm = Teuchos::TimeMonitor::getNewTimer(timerName);
tm->start();
bool hasParamList = inParamList.numParams();
RCP<HierarchyManager> mueLuFactory;
// Rip off non-serializable data before validation
Teuchos::ParameterList nonSerialList,paramList;
MueLu::ExtractNonSerializableData(inParamList, paramList, nonSerialList);
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter(paramList,op->getDomainMap()->getComm()));
}
// Create Hierarchy
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(op->getDomainMap()->lib());
// Stick the non-serializible data on the hierarchy.
HierarchyUtils::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
// Set fine level operator
H->GetLevel(0)->Set("A", op);
// Set coordinates if available
if (coords != Teuchos::null) {
H->GetLevel(0)->Set("Coordinates", coords);
}
// Wrap nullspace if available, otherwise use constants
if (nullspace == Teuchos::null) {
int nPDE = MueLu::MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = MultiVectorFactory::Build(op->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<Scalar>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<Scalar> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GlobalOrdinal GID = op->getDomainMap()->getGlobalElement(j) - op->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<Scalar>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
mueLuFactory->SetupHierarchy(*H);
tm->stop();
tm->incrementNumCalls();
if (H->GetVerbLevel() & Statistics0) {
const bool alwaysWriteLocal = true;
const bool writeGlobalStats = true;
const bool writeZeroTimers = false;
const bool ignoreZeroTimers = true;
const std::string filter = timerName;
Teuchos::TimeMonitor::summarize(op->getRowMap()->getComm().ptr(), std::cout, alwaysWriteLocal, writeGlobalStats,
writeZeroTimers, Teuchos::Union, filter, ignoreZeroTimers);
}
tm->reset();
return H;
}
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> > &inA,
Teuchos::ParameterList& inParamList,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node>>& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>>& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
typedef Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> crs_matrix_type;
typedef Tpetra::Experimental::BlockCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> block_crs_matrix_type;
bool hasParamList = inParamList.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = inParamList;
RCP<const crs_matrix_type> constCrsA;
RCP<crs_matrix_type> crsA;
#if defined(HAVE_MUELU_EXPERIMENTAL) and defined(HAVE_MUELU_AMGX)
std::string externalMG = "use external multigrid package";
if (hasParamList && paramList.isParameter(externalMG) && paramList.get<std::string>(externalMG) == "amgx"){
constCrsA = rcp_dynamic_cast<const crs_matrix_type>(inA);
TEUCHOS_TEST_FOR_EXCEPTION(constCrsA == Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: failed to dynamic cast to Tpetra::CrsMatrix, which is required to be able to use AmgX.");
return rcp(new AMGXOperator<SC,LO,GO,NO>(inA,inParamList));
}
#endif
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList,inA->getDomainMap()->getComm()));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A;
RCP<block_crs_matrix_type> bcrsA = rcp_dynamic_cast<block_crs_matrix_type>(inA);
crsA = rcp_dynamic_cast<crs_matrix_type>(inA);
if (crsA != Teuchos::null)
A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(crsA);
else if (bcrsA != Teuchos::null) {
RCP<Xpetra::CrsMatrix<SC,LO,GO,NO> > temp = rcp(new Xpetra::TpetraBlockCrsMatrix<SC,LO,GO,NO>(bcrsA));
TEUCHOS_TEST_FOR_EXCEPTION(temp==Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: cast from Tpetra::Experimental::BlockCrsMatrix to Xpetra::TpetraBlockCrsMatrix failed.");
A = rcp(new Xpetra::CrsMatrixWrap<SC,LO,GO,NO>(temp));
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "CreateTpetraPreconditioner: only Tpetra CrsMatrix and BlockCrsMatrix types are supported.");
}
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
int main(int argc, char *argv[]) {
#include "MueLu_UseShortNames.hpp"
using Teuchos::RCP; using Teuchos::rcp;
using Teuchos::TimeMonitor;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole);
bool success = false;
bool verbose = true;
try {
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
Teuchos::CommandLineProcessor clp(false);
std::string xmlFileName = "TwoBillion.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file");
int num_per_proc = 1000; clp.setOption("dpc", &num_per_proc, "DOFs per core");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
int NumProcs = comm->getSize();
int MyPID = comm->getRank();
if(!MyPID) printf("TwoBillion: Running Test\n");
const long long FIRST_GID = 3000000000L;
// const long long FIRST_GID = 0L;
const long long IndexBase = 0L;
// const long long IndexBase = 3000000000L;
global_size_t NumGlobalElements = NumProcs*num_per_proc;
// Create Map w/ GIDs starting at > 2 billion
RCP<const Map> map;
RCP<CrsMatrix> Acrs;
Teuchos::Array<GlobalOrdinal> mygids(num_per_proc);
for(int i=0; i<num_per_proc; i++)
mygids[i] = FIRST_GID + MyPID*num_per_proc + i;
for (int i=0; i<NumProcs; ++i) {
if (i==MyPID)
std::cout << "pid " << i << " : 1st GID = " << mygids[0] << std::endl;
}
//for(int i=0;i<num_per_proc;i++)
// printf("[%d] mygids[%d] = %lld\n",MyPID,i,mygids[i]);
map = MapFactory::Build(Xpetra::UseTpetra, Teuchos::OrdinalTraits<global_size_t>::invalid(),mygids(),IndexBase,comm);
// RCP<Teuchos::FancyOStream> fox = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
// fox->setOutputToRootOnly(-1);
// map->describe(*fox,Teuchos::VERB_EXTREME);
// Create 1D Laplacian w/ GIDs starting at > 2 billion
Teuchos::Array<Scalar> myvals(3);
Teuchos::Array<GlobalOrdinal> mycols(3);
Teuchos::ArrayView<Scalar> ValView;
Teuchos::ArrayView<GlobalOrdinal> ColView;
Acrs = CrsMatrixFactory::Build(map,3);
for(int i=0; i<num_per_proc; i++) {
if(mygids[i]==FIRST_GID ) {
mycols[0] = mygids[i]; myvals[0] = 2;
mycols[1] = mygids[i]+1; myvals[1] = -1;
ValView=myvals.view(0,2);
ColView=mycols.view(0,2);
// printf("[%d %lld] cols %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1]);
}
else if(mygids[i] == FIRST_GID + (long long) NumGlobalElements - 1){
mycols[0] = mygids[i]-1; myvals[0] = -1;
mycols[1] = mygids[i]; myvals[1] = 2;
ValView=myvals.view(0,2);
ColView=mycols.view(0,2);
// printf("[%d %lld] cols %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1]);
}
else {
mycols[0] = mygids[i]-1; myvals[0] = -1;
mycols[1] = mygids[i]; myvals[1] = -2;
mycols[2] = mygids[i]+1; myvals[1] = -1;
ValView=myvals();
ColView=mycols();
// printf("[%d %lld] cols %lld %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1],mycols[2]);
}
Acrs->insertGlobalValues(mygids[i],ColView,ValView);
}
Acrs->fillComplete();
RCP<Matrix> A = rcp(new CrsMatrixWrap(Acrs));
RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
RCP<MultiVector> coordinates = MultiVectorFactory::Build(map, 1);
Teuchos::ArrayRCP<Scalar> coordVals = coordinates->getDataNonConst(0);
double h = 1.0 / NumGlobalElements;
for (LocalOrdinal i=0; i<num_per_proc; ++i)
coordVals[i] = MyPID*num_per_proc*h + i*h;
coordVals = Teuchos::null;
Teuchos::ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
RCP<HierarchyManager> mueLuFactory;
mueLuFactory = rcp(new ParameterListInterpreter(xmlFileName, *comm));
RCP<Hierarchy> H;
H = mueLuFactory->CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory->SetupHierarchy(*H);
//
//
// SOLVE
//
//
// Define X, B
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
X->setSeed(846930886);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
B->norm2(norms);
B->scale(1.0/norms[0]);
//
// Use AMG as a preconditioner in Belos
//
#ifdef HAVE_MUELU_BELOS
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO>(A)); // Turns a Xpetra::Operator object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setLeftPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
int maxIts = 100;
double optTol = 1e-8;
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", optTol); // Relative convergence tolerance requested
//belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
if (comm->getRank() == 0)
std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
if (solver->getNumIters() > 6) {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
return(EXIT_FAILURE);
}
// Compute actual residuals.
int numrhs = 1;
std::vector<double> actual_resids( numrhs ); //TODO: double?
std::vector<double> rhs_norm( numrhs );
RCP<MultiVector> resid = MultiVectorFactory::Build(map, numrhs);
typedef Belos::OperatorTraits<SC, MV, OP> OPT;
typedef Belos::MultiVecTraits<SC, MV> MVT;
OPT::Apply( *belosOp, *X, *resid );
MVT::MvAddMv( -1.0, *resid, 1.0, *B, *resid );
MVT::MvNorm( *resid, actual_resids );
MVT::MvNorm( *B, rhs_norm );
*out<< "---------- Actual Residuals (normalized) ----------"<<std::endl<<std::endl;
for ( int i = 0; i<numrhs; i++) {
double actRes = actual_resids[i]/rhs_norm[i];
*out<<"Problem "<<i<<" : \t"<< actRes <<std::endl;
//if (actRes > tol) { badRes = true; }
}
} //try
catch(...) {
if (comm->getRank() == 0)
std::cout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
success = (ret == Belos::Converged);
// Check convergence
if (success) {
if (comm->getRank() == 0) std::cout << std::endl << "SUCCESS: Belos converged!" << std::endl;
} else {
if (comm->getRank() == 0) std::cout << std::endl << "ERROR: Belos did not converge! " << std::endl;
}
#endif // HAVE_MUELU_BELOS
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
using Teuchos::ParameterList;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100, nz = 100;
Galeri::Xpetra::Parameters<GO> galeriParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file [default = 'scalingTest.xml']");
bool printTimings = true; clp.setOption("timings", "notimings", &printTimings, "print timings to screen");
int writeMatricesOPT = -2; clp.setOption("write", &writeMatricesOPT, "write matrices to file (-1 means all; i>=0 means level i)");
std::string dsolveType = "cg", solveType; clp.setOption("solver", &dsolveType, "solve type: (none | cg | gmres | standalone)");
double dtol = 1e-12, tol; clp.setOption("tol", &dtol, "solver convergence tolerance");
std::string mapFile; clp.setOption("map", &mapFile, "map data file");
std::string matrixFile; clp.setOption("matrix", &matrixFile, "matrix data file");
std::string coordFile; clp.setOption("coords", &coordFile, "coordinates data file");
int numRebuilds = 0; clp.setOption("rebuild", &numRebuilds, "#times to rebuild hierarchy");
int maxIts = 200; clp.setOption("its", &maxIts, "maximum number of solver iterations");
bool scaleResidualHistory = true; clp.setOption("scale", "noscale", &scaleResidualHistory, "scaled Krylov residual history");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<ParameterList>(¶mList), *comm);
bool isDriver = paramList.isSublist("Run1");
if (isDriver) {
// update galeriParameters with the values from the XML file
ParameterList& realParams = galeriParameters.GetParameterList();
for (ParameterList::ConstIterator it = realParams.begin(); it != realParams.end(); it++) {
const std::string& name = realParams.name(it);
if (paramList.isParameter(name))
realParams.setEntry(name, paramList.getEntry(name));
}
}
// Retrieve matrix parameters (they may have been changed on the command line)
// [for instance, if we changed matrix type from 2D to 3D we need to update nz]
ParameterList galeriList = galeriParameters.GetParameterList();
// =========================================================================
// Problem construction
// =========================================================================
std::ostringstream galeriStream;
comm->barrier();
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
RCP<TimeMonitor> tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<Matrix> A;
RCP<const Map> map;
RCP<MultiVector> coordinates;
RCP<MultiVector> nullspace;
if (matrixFile.empty()) {
galeriStream << "========================================================\n" << xpetraParameters << galeriParameters;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
std::string matrixType = galeriParameters.GetMatrixType();
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriList);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriList);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriList);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
galeriStream << "Processor subdomains in x direction: " << galeriList.get<int>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriList.get<int>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriList.get<int>("mz") << std::endl
<< "========================================================" << std::endl;
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") {
// Our default test case for elasticity: all boundaries of a square/cube have Neumann b.c. except left which has Dirichlet
galeriList.set("right boundary" , "Neumann");
galeriList.set("bottom boundary", "Neumann");
galeriList.set("top boundary" , "Neumann");
galeriList.set("front boundary" , "Neumann");
galeriList.set("back boundary" , "Neumann");
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(galeriParameters.GetMatrixType(), map, galeriList);
A = Pr->BuildMatrix();
nullspace = MultiVectorFactory::Build(map, 1);
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((galeriParameters.GetMatrixType() == "Elasticity2D") ? 2 : 3);
} else {
nullspace->putScalar(one);
}
} else {
if (!mapFile.empty())
map = Utils2::ReadMap(mapFile, xpetraParameters.GetLib(), comm);
comm->barrier();
if (lib == Xpetra::UseEpetra) {
A = Utils::Read(matrixFile, map);
} else {
// Tpetra matrix reader is still broken, so instead we read in
// a matrix in a binary format and then redistribute it
const bool binaryFormat = true;
A = Utils::Read(matrixFile, lib, comm, binaryFormat);
RCP<Matrix> newMatrix = MatrixFactory::Build(map, 1);
RCP<Import> importer = ImportFactory::Build(A->getRowMap(), map);
newMatrix->doImport(*A, *importer, Xpetra::INSERT);
newMatrix->fillComplete();
A.swap(newMatrix);
}
comm->barrier();
if (!coordFile.empty())
coordinates = Utils2::ReadMultiVector(coordFile, map);
nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(one);
}
comm->barrier();
tm = Teuchos::null;
galeriStream << "Galeri complete.\n========================================================" << std::endl;
int numReruns = 1;
if (paramList.isParameter("number of reruns"))
numReruns = paramList.get<int>("number of reruns");
const bool mustAlreadyExist = true;
for (int rerunCount = 1; rerunCount <= numReruns; rerunCount++) {
ParameterList mueluList, runList;
bool stop = false;
if (isDriver) {
runList = paramList.sublist("Run1", mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} else {
mueluList = paramList;
stop = true;
}
int runCount = 1;
do {
A->SetMaxEigenvalueEstimate(-one);
solveType = dsolveType;
tol = dtol;
int savedOut = -1;
FILE* openedOut = NULL;
if (isDriver) {
if (runList.isParameter("filename")) {
// Redirect all output into a filename We have to redirect all output,
// including printf's, therefore we cannot simply replace C++ cout
// buffers, and have to use heavy machinary (dup2)
std::string filename = runList.get<std::string>("filename");
if (numReruns > 1)
filename += "_run" + MueLu::toString(rerunCount);
filename += (lib == Xpetra::UseEpetra ? ".epetra" : ".tpetra");
savedOut = dup(STDOUT_FILENO);
openedOut = fopen(filename.c_str(), "w");
dup2(fileno(openedOut), STDOUT_FILENO);
}
if (runList.isParameter("solver")) solveType = runList.get<std::string>("solver");
if (runList.isParameter("tol")) tol = runList.get<double> ("tol");
}
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
fancyout << galeriStream.str();
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
RCP<HierarchyManager> mueLuFactory = rcp(new ParameterListInterpreter(mueluList));
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
RCP<Hierarchy> H;
for (int i = 0; i <= numRebuilds; i++) {
A->SetMaxEigenvalueEstimate(-one);
H = mueLuFactory->CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
if (!coordinates.is_null())
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory->SetupHierarchy(*H);
}
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
{
// we set seed for reproducibility
Utils::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
tm = Teuchos::null;
if (writeMatricesOPT > -2) {
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3.5 - Matrix output")));
H->Write(writeMatricesOPT, writeMatricesOPT);
tm = Teuchos::null;
}
comm->barrier();
if (solveType == "none") {
// Do not perform a solve
} else if (solveType == "standalone") {
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Fixed Point Solve")));
H->IsPreconditioner(false);
H->Iterate(*B, *X, maxIts);
} else if (solveType == "cg" || solveType == "gmres") {
#ifdef HAVE_MUELU_BELOS
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp <SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
fancyout << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
if (!scaleResidualHistory)
belosList.set("Implicit Residual Scaling", "None");
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg") {
solver = rcp(new Belos::PseudoBlockCGSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
} else if (solveType == "gmres") {
solver = rcp(new Belos::BlockGmresSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
}
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
fancyout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
} catch(...) {
fancyout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged)
fancyout << std::endl << "ERROR: Belos did not converge! " << std::endl;
else
fancyout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif //ifdef HAVE_MUELU_BELOS
} else {
throw MueLu::Exceptions::RuntimeError("Unknown solver type: \"" + solveType + "\"");
}
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union);
TimeMonitor::clearCounters();
if (isDriver) {
if (openedOut != NULL) {
dup2(savedOut, STDOUT_FILENO);
fclose(openedOut);
openedOut = NULL;
}
try {
runList = paramList.sublist("Run" + MueLu::toString(++runCount), mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} catch (std::exception) {
stop = true;
}
}
} while (stop == false);
}
return 0;
} //main
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::CrsMatrix <Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inA,
Teuchos::ParameterList& paramList,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
bool hasParamList = paramList.numParams();
RCP<HierarchyManager> mueLuFactory;
RCP<Hierarchy> H;
if (hasParamList) {
mueLuFactory = rcp(new ParameterListInterpreter<SC,LO,GO,NO>(paramList));
H = mueLuFactory->CreateHierarchy();
} else {
H = rcp(new Hierarchy());
}
// Wrap A
RCP<Matrix> A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(inA);
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<MultiVector> coordinates = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = 1;
if (paramList.isSublist("Matrix")) {
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
if (hasParamList)
mueLuFactory->SetupHierarchy(*H);
else
H->Setup();
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}