Mesh MeshBuilder::createMesh(const ParameterList& params)
{
TEST_FOR_EXCEPTION(!params.isParameter("type"), RuntimeError,
"field name 'type' expected but not found in MeshBuilder "
"input parameter list: " << params);
std::string type = params.get<string>("type");
MeshSource mesher;
if (type=="Rectangle")
{
mesher = new PartitionedRectangleMesher(params);
}
else if (type=="Line")
{
mesher = new PartitionedLineMesher(params);
}
else if (type=="Exodus")
{
mesher = new ExodusNetCDFMeshReader(params);
}
else if (type=="Triangle")
{
mesher = new TriangleMeshReader(params);
}
TEST_FOR_EXCEPTION(mesher.ptr().get()==0, RuntimeError,
"null mesh source in MeshBuilder::createMesh()");
return mesher.getMesh();
}
template <typename T> inline
void LinearSolverBase<Scalar>::setParameter(const ParameterList& params,
T* dataPtr,
const std::string& name)
{
if (!params.isParameter(name)) return;
TEUCHOS_TEST_FOR_EXCEPTION(!params.template isType<T>(name), std::runtime_error,
"invalid type for parameter [" << name << "]");
*dataPtr = params.template get<T>(name);
}
void set_ibcs(
ParameterList const& p,
RCP<Integrator> w,
SolInfo* s) {
if (! p.isParameter("side set")) return;
validate_params(p, s);
apf::Vector3 center(0,0,0);
center[0] = p.get<double>("x_0");
center[1] = p.get<double>("y_0");
center[2] = p.get<double>("z_0");
auto scale = p.get<double>("scale distance");
auto ss_name = p.get<std::string>("side set");
auto ww = rcp_static_cast<VectorWeight>(w);
apply_bc(scale, center, ss_name, ww, s);
}
DefaultOptConvergenceTest::DefaultOptConvergenceTest(
const ParameterList& params)
: OptConvergenceTestBase(params),
minIters_(params.get<int>("Min Iterations")),
maxIters_(params.get<int>("Max Iterations")),
requiredPasses_(params.get<int>("Num Required Passes")),
objTol_(params.get<double>("Objective Tolerance")),
gradTol_(params.get<double>("Gradient Tolerance")),
stepTol_(params.get<double>("Step Tolerance")),
xTyp_(1.0),
fTyp_(1.0)
{
if (params.isParameter("Typical X Scale")) xTyp_ = params.get<double>("Typical X Scale");
if (params.isParameter("Typical F Scale")) fTyp_ = params.get<double>("Typical F Scale");
}
NonlinearSolver<double> NonlinearSolverBuilder::createSolver(const ParameterList& params)
{
if (params.isSublist("NOX Solver"))
{
return new NOXSolver(params);
}
else if (params.isSublist("Nonlinear Solver"))
{
ParameterList sub = params.sublist("Nonlinear Solver");
Array<string> names = tuple<string>("Newton Armijo Solver", "Newton-Armijo Solver", "NewtonArmijoSolver");
for (int i=0; i<names.size(); i++)
{
if (sub.isSublist(names[i]))
{
ParameterList subsub = sub.sublist(names[i]);
LinearSolver<double> linSolver;
if (subsub.isParameter("Linear Solver"))
{
string solverFile = subsub.get<string>("Linear Solver");
linSolver = LinearSolverBuilder::createSolver(solverFile);
}
else if (subsub.isSublist("Linear Solver"))
{
linSolver = LinearSolverBuilder::createSolver(subsub);
}
else
{
TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
"Nonlinear solver parameter list " << sub
<< " does not appear to specify a solver for the linear subproblems");
}
return new NewtonArmijoSolver<double>(subsub, linSolver);
}
}
}
else
{
TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
"Nonlinear solver parameter list " << params
<< " can't be parsed to find a nonlinear solver");
}
return NonlinearSolver<double>();
}
void ParameterListInterpreter<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::UpdateFactoryManager(Teuchos::ParameterList& paramList,
const Teuchos::ParameterList& defaultList, FactoryManager& manager) const {
// NOTE: Factory::SetParameterList must be called prior to Factory::SetFactory, as
// SetParameterList sets default values for non mentioned parameters, including factories
// === Smoothing ===
bool isCustomSmoother =
paramList.isParameter("smoother: pre or post") ||
paramList.isParameter("smoother: type") || paramList.isParameter("smoother: pre type") || paramList.isParameter("smoother: post type") ||
paramList.isSublist ("smoother: params") || paramList.isSublist ("smoother: pre params") || paramList.isSublist ("smoother: post params") ||
paramList.isParameter("smoother: sweeps") || paramList.isParameter("smoother: pre sweeps") || paramList.isParameter("smoother: post sweeps") ||
paramList.isParameter("smoother: overlap") || paramList.isParameter("smoother: pre overlap") || paramList.isParameter("smoother: post overlap");;
MUELU_READ_2LIST_PARAM(paramList, defaultList, "smoother: pre or post", std::string, "both", PreOrPost);
if (PreOrPost == "none") {
manager.SetFactory("Smoother", Teuchos::null);
} else if (isCustomSmoother) {
// FIXME: get default values from the factory
// NOTE: none of the smoothers at the moment use parameter validation framework, so we
// cannot get the default values from it.
#define TEST_MUTUALLY_EXCLUSIVE(arg1,arg2) \
TEUCHOS_TEST_FOR_EXCEPTION(paramList.isParameter(#arg1) && paramList.isParameter(#arg2), \
Exceptions::InvalidArgument, "You cannot specify both \""#arg1"\" and \""#arg2"\"");
#define TEST_MUTUALLY_EXCLUSIVE_S(arg1,arg2) \
TEUCHOS_TEST_FOR_EXCEPTION(paramList.isSublist(#arg1) && paramList.isSublist(#arg2), \
Exceptions::InvalidArgument, "You cannot specify both \""#arg1"\" and \""#arg2"\"");
TEST_MUTUALLY_EXCLUSIVE ("smoother: type", "smoother: pre type");
TEST_MUTUALLY_EXCLUSIVE ("smoother: type", "smoother: post type");
TEST_MUTUALLY_EXCLUSIVE ("smoother: sweeps", "smoother: pre sweeps");
TEST_MUTUALLY_EXCLUSIVE ("smoother: sweeps", "smoother: post sweeps");
TEST_MUTUALLY_EXCLUSIVE ("smoother: overlap", "smoother: pre overlap");
TEST_MUTUALLY_EXCLUSIVE ("smoother: overlap", "smoother: post overlap");
TEST_MUTUALLY_EXCLUSIVE_S("smoother: params", "smoother: pre params");
TEST_MUTUALLY_EXCLUSIVE_S("smoother: params", "smoother: post params");
TEUCHOS_TEST_FOR_EXCEPTION(PreOrPost == "both" && (paramList.isParameter("smoother: pre type") != paramList.isParameter("smoother: post type")),
Exceptions::InvalidArgument, "You must specify both \"smoother: pre type\" and \"smoother: post type\"");
// Default values
int overlap = 0;
ParameterList defaultSmootherParams;
defaultSmootherParams.set("relaxation: type", "Symmetric Gauss-Seidel");
defaultSmootherParams.set("relaxation: sweeps", Teuchos::OrdinalTraits<LO>::one());
defaultSmootherParams.set("relaxation: damping factor", Teuchos::ScalarTraits<Scalar>::one());
RCP<SmootherPrototype> preSmoother = Teuchos::null, postSmoother = Teuchos::null;
std::string preSmootherType, postSmootherType;
ParameterList preSmootherParams, postSmootherParams;
if (paramList.isParameter("smoother: overlap"))
overlap = paramList.get<int>("smoother: overlap");
if (PreOrPost == "pre" || PreOrPost == "both") {
if (paramList.isParameter("smoother: pre type")) {
preSmootherType = paramList.get<std::string>("smoother: pre type");
} else {
MUELU_READ_2LIST_PARAM(paramList, defaultList, "smoother: type", std::string, "RELAXATION", preSmootherTypeTmp);
preSmootherType = preSmootherTypeTmp;
}
if (paramList.isParameter("smoother: pre overlap"))
overlap = paramList.get<int>("smoother: pre overlap");
if (paramList.isSublist("smoother: pre params"))
preSmootherParams = paramList.sublist("smoother: pre params");
else if (paramList.isSublist("smoother: params"))
preSmootherParams = paramList.sublist("smoother: params");
else if (defaultList.isSublist("smoother: params"))
preSmootherParams = defaultList.sublist("smoother: params");
else if (preSmootherType == "RELAXATION")
preSmootherParams = defaultSmootherParams;
preSmoother = rcp(new TrilinosSmoother(preSmootherType, preSmootherParams, overlap));
}
if (PreOrPost == "post" || PreOrPost == "both") {
if (paramList.isParameter("smoother: post type"))
postSmootherType = paramList.get<std::string>("smoother: post type");
else {
MUELU_READ_2LIST_PARAM(paramList, defaultList, "smoother: type", std::string, "RELAXATION", postSmootherTypeTmp);
postSmootherType = postSmootherTypeTmp;
}
if (paramList.isSublist("smoother: post params"))
postSmootherParams = paramList.sublist("smoother: post params");
else if (paramList.isSublist("smoother: params"))
postSmootherParams = paramList.sublist("smoother: params");
else if (defaultList.isSublist("smoother: params"))
postSmootherParams = defaultList.sublist("smoother: params");
else if (postSmootherType == "RELAXATION")
postSmootherParams = defaultSmootherParams;
if (paramList.isParameter("smoother: post overlap"))
overlap = paramList.get<int>("smoother: post overlap");
if (postSmootherType == preSmootherType && areSame(preSmootherParams, postSmootherParams))
postSmoother = preSmoother;
else
postSmoother = rcp(new TrilinosSmoother(postSmootherType, postSmootherParams, overlap));
}
manager.SetFactory("Smoother", rcp(new SmootherFactory(preSmoother, postSmoother)));
}
// === Coarse solver ===
bool isCustomCoarseSolver =
paramList.isParameter("coarse: type") ||
paramList.isParameter("coarse: params");
if (paramList.isParameter("coarse: type") && paramList.get<std::string>("coarse: type") == "none") {
manager.SetFactory("CoarseSolver", Teuchos::null);
} else if (isCustomCoarseSolver) {
// FIXME: get default values from the factory
// NOTE: none of the smoothers at the moment use parameter validation framework, so we
// cannot get the default values from it.
MUELU_READ_2LIST_PARAM(paramList, defaultList, "coarse: type", std::string, "", coarseType);
ParameterList coarseParams;
if (paramList.isSublist("coarse: params"))
coarseParams = paramList.sublist("coarse: params");
else if (defaultList.isSublist("coarse: params"))
coarseParams = defaultList.sublist("coarse: params");
RCP<SmootherPrototype> coarseSmoother;
// TODO: this is not a proper place to check. If we consider direct solver to be a special
// case of smoother, we would like to unify Amesos and Ifpack2 smoothers in src/Smoothers, and
// have a single factory responsible for those. Then, this check would belong there.
if (coarseType == "RELAXATION" || coarseType == "CHEBYSHEV" ||
coarseType == "ILUT" || coarseType == "ILU" || coarseType == "RILUK" || coarseType == "SCHWARZ")
coarseSmoother = rcp(new TrilinosSmoother(coarseType, coarseParams));
else
coarseSmoother = rcp(new DirectSolver(coarseType, coarseParams));
manager.SetFactory("CoarseSolver", rcp(new SmootherFactory(coarseSmoother)));
}
// === Aggregation ===
// Aggregation graph
RCP<CoalesceDropFactory> dropFactory = rcp(new CoalesceDropFactory());
ParameterList dropParams;
dropParams.set("lightweight wrap", true);
MUELU_TEST_AND_SET_PARAM(dropParams, "algorithm", paramList, defaultList, "aggregation: drop scheme", std::string);
// Rename classical to original
if (dropParams.isParameter("algorithm") && dropParams.get<std::string>("algorithm") == "classical")
dropParams.set("algorithm", "original");
MUELU_TEST_AND_SET_PARAM(dropParams, "aggregation threshold", paramList, defaultList, "aggregation: drop tol", double);
MUELU_TEST_AND_SET_PARAM(dropParams, "Dirichlet detection threshold", paramList, defaultList, "aggregation: Dirichlet threshold", double);
dropFactory->SetParameterList(dropParams);
manager.SetFactory("Graph", dropFactory);
// Aggregation sheme
MUELU_READ_2LIST_PARAM(paramList, defaultList, "aggregation: type", std::string, "uncoupled", aggType);
RCP<Factory> aggFactory;
if (aggType == "uncoupled") {
aggFactory = rcp(new UncoupledAggregationFactory());
ParameterList aggParams;
MUELU_TEST_AND_SET_PARAM(aggParams, "mode", paramList, defaultList, "aggregation: mode", std::string);
MUELU_TEST_AND_SET_PARAM(aggParams, "MinNodesPerAggregate", paramList, defaultList, "aggregation: min agg size", int);
MUELU_TEST_AND_SET_PARAM(aggParams, "MaxNodesPerAggregate", paramList, defaultList, "aggregation: max agg size", int);
MUELU_TEST_AND_SET_PARAM(aggParams, "aggregation: preserve Dirichlet points", paramList, defaultList, "aggregation: preserve Dirichlet points", bool);
aggFactory->SetParameterList(aggParams);
} else if (aggType == "coupled") {
void ParameterListInterpreter<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::SetEasyParameterList(const Teuchos::ParameterList& constParamList) {
// Create a non const copy of the parameter list
// Working with a modifiable list is much much easier than with original one
ParameterList paramList = constParamList;
// Translate cycle type parameter
if (paramList.isParameter("cycle type")) {
std::map<std::string,CycleType> cycleMap;
cycleMap["V"] = VCYCLE;
cycleMap["W"] = WCYCLE;
std::string cycleType = paramList.get<std::string>("cycle type");
TEUCHOS_TEST_FOR_EXCEPTION(cycleMap.count(cycleType) == 0, Exceptions::RuntimeError, "Invalid cycle type: \"" << cycleType << "\"");
Cycle_ = cycleMap[cycleType];
}
this->maxCoarseSize_ = paramList.get<int> ("coarse: max size", Hierarchy::GetDefaultMaxCoarseSize());
this->numDesiredLevel_ = paramList.get<int> ("max levels", Hierarchy::GetDefaultMaxLevels());
this->graphOutputLevel_ = paramList.get<int> ("debug: graph level", -1);
blockSize_ = paramList.get<int> ("number of equations", 1);
// Save level data
if (paramList.isSublist("print")) {
ParameterList printList = paramList.sublist("print");
if (printList.isParameter("A"))
this->matricesToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "A");
if (printList.isParameter("P"))
this->prolongatorsToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "P");
if (printList.isParameter("R"))
this->restrictorsToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "R");
}
// Translate verbosity parameter
this->verbosity_ = static_cast<MsgType>(Hierarchy::GetDefaultVerbLevel()); // cast int to enum
if (paramList.isParameter("verbosity")) {
std::map<std::string,MsgType> verbMap;
verbMap["none"] = None;
verbMap["low"] = Low;
verbMap["medium"] = Medium;
verbMap["high"] = High;
verbMap["extreme"] = Extreme;
verbMap["test"] = Test;
std::string verbosityLevel = paramList.get<std::string>("verbosity");
TEUCHOS_TEST_FOR_EXCEPTION(verbMap.count(verbosityLevel) == 0, Exceptions::RuntimeError, "Invalid verbosity level: \"" << verbosityLevel << "\"");
this->verbosity_ = verbMap[verbosityLevel];
this->SetVerbLevel(this->verbosity_);
}
// Detect if we need to transfer coordinates to coarse levels. We do that iff
// - we use "laplacian" dropping on some level, or
// - we use repartitioning on some level
// This is not ideal, as we may have "repartition: enable" turned on by default
// and not present in the list, but it is better than nothing.
useCoordinates_ = false;
if ((paramList.isParameter("repartition: enable") && paramList.get<bool>("repartition: enable") == true) ||
(paramList.isParameter("aggregation: drop scheme") && paramList.get<std::string>("aggregation: drop scheme") == "laplacian")) {
useCoordinates_ = true;
} else {
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
std::string levelStr = "level" + toString(levelID);
if (paramList.isSublist(levelStr)) {
const ParameterList& levelList = paramList.sublist(levelStr);
if ((levelList.isParameter("repartition: enable") && levelList.get<bool>("repartition: enable") == true) ||
(levelList.isParameter("aggregation: drop scheme") && levelList.get<std::string>("aggregation: drop scheme") == "laplacian")) {
useCoordinates_ = true;
break;
}
}
}
}
// Detect if we do implicit P and R rebalance
if (paramList.isParameter("repartition: enable") && paramList.get<bool>("repartition: enable") == true)
this->doPRrebalance_ = paramList.get<bool>("repartition: rebalance P and R", Hierarchy::GetDefaultPRrebalance());
this->implicitTranspose_ = paramList.get<bool>("transpose: use implicit", Hierarchy::GetDefaultImplicitTranspose());
// Create default manager
RCP<FactoryManager> defaultManager = rcp(new FactoryManager());
defaultManager->SetVerbLevel(this->verbosity_);
UpdateFactoryManager(paramList, ParameterList(), *defaultManager);
defaultManager->Print();
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
RCP<FactoryManager> levelManager;
if (paramList.isSublist("level " + toString(levelID))) {
// Some level specific parameters, update default manager
bool mustAlreadyExist = true;
ParameterList& levelList = paramList.sublist("level " + toString(levelID), mustAlreadyExist);
levelManager = rcp(new FactoryManager(*defaultManager));
levelManager->SetVerbLevel(defaultManager->GetVerbLevel());
UpdateFactoryManager(levelList, paramList, *levelManager);
} else {
// No level specific parameter, use default manager
levelManager = defaultManager;
}
this->AddFactoryManager(levelID, 1, levelManager);
}
if (paramList.isParameter("strict parameter checking") &&
paramList.get<bool> ("strict parameter checking")) {
ParameterList unusedParamList;
// Check for unused parameters that aren't lists
for (ParameterList::ConstIterator itr = paramList.begin(); itr != paramList.end(); ++itr) {
const ParameterEntry& entry = paramList.entry(itr);
if (!entry.isList() && !entry.isUsed())
unusedParamList.setEntry(paramList.name(itr), entry);
}
#if 0
// Check for unused parameters in level-specific sublists
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
std::string levelStr = "level" + toString(levelID);
if (paramList.isSublist(levelStr)) {
const ParameterList& levelList = paramList.sublist(levelStr);
for (ParameterList::ConstIterator itr = levelList.begin(); itr != levelList.end(); ++itr) {
const ParameterEntry& entry = levelList.entry(itr);
if (!entry.isList() && !entry.isUsed())
unusedParamList.sublist(levelStr).setEntry(levelList.name(itr), entry);
}
}
}
#endif
if (unusedParamList.numParams() > 0) {
std::ostringstream unusedParamsStream;
int indent = 4;
unusedParamList.print(unusedParamsStream, indent);
TEUCHOS_TEST_FOR_EXCEPTION_PURE_MSG(true, Teuchos::Exceptions::InvalidParameter,
"WARNING: Unused parameters were detected. Please check spelling and type." << std::endl << unusedParamsStream.str());
}
}
// FIXME: parameters passed to packages, like Ifpack2, are not touched by us, resulting in "[unused]" flag
// being displayed. On the other hand, we don't want to simply iterate through them touching. I don't know
// what a good solution looks like
this->GetOStream(static_cast<MsgType>(Runtime1 | Test), 0) << paramList << std::endl;
}
bool ComparisonHelper::Compare(const ParameterList &pList, const RCP<const Comm<int> > &comm)
{
if(pList.isParameter("A") && pList.isParameter("B")) {
// comparing solutions
string pA = pList.get<string>("A");
if(this->sources.find(pA) == this->sources.end())
{
cout << "\nProblem: " + pA + ", was not saved for comparison.";
cout << "\nThis typically indicates that an error occurred while running the problem.";
cout << "\nSolution comparison FAILED." << endl;
return false;
}
string pB = pList.get<string>("B");
if(this->sources.find(pB) == this->sources.end()) {
cout << "\nProblem: " + pB + ", was not saved for comparison.";
cout << "\nThis typically indicates that an error occurred while running the problem.";
cout << "\nSolution comparison FAILED." << endl;
return false;
}
bool bResult = this->CompareSolutions(pA, pB, comm);
return bResult;
}
else if (pList.isParameter("Problem") && pList.isParameter("Reference")) {
// comparing metrics/timers
string prb = pList.get<string>("Problem");
if(this->sources.find(prb) == this->sources.end()) {
cout << "\nProblem: " + prb + ", was not saved for comparison.";
cout << "\nThis typically indicates that an error occurred while running the problem.";
cout << "\nMetric comparison FAILED." << endl;
return false;
}
string ref = pList.get<string>("Reference");
if(this->sources.find(ref) == this->sources.end()) {
cout << "\nReference: " + ref + ", was not saved for comparison.";
cout << "\nThis typically indicates that an error occurred while running the problem.";
cout << "\nMetric comparison FAILED." << endl;
return false;
}
bool bResult = this->CompareMetrics(pList, comm);
return bResult;
}
else if (pList.isParameter("A") || pList.isParameter("B"))
{
if(comm->getRank() == 0)
{
cout << "Problem A or Problem B is not specified -- check input.";
cout <<"\nSolution comparison FAILED." << endl;
}
}
else if (pList.isParameter("Problem") || pList.isParameter("Reference")) {
if(comm->getRank() == 0) {
cout << "Problem or reference is not specified -- check input.";
cout <<"\nMetric comparison FAILED." << endl;
}
}
else {
if (comm->getRank() == 0) {
cout << "ComparisonHelper did not understand how to read the xml. Test FAILED." << endl;
}
}
return false;
}
void run(const UserInputForTests &uinput,
const ParameterList &problem_parameters,
RCP<ComparisonHelper> & comparison_helper,
const RCP<const Teuchos::Comm<int> > & comm)
{
// Major steps in running a problem in zoltan 2
// 1. get an input adapter
// 2. construct the problem
// 3. solve the problem
// 4. analyze metrics
// 5. clean up
int rank = comm->getRank();
if(!problem_parameters.isParameter("kind"))
{
if(rank == 0) std::cerr << "Problem kind not provided" << std::endl;
return;
}
if(!problem_parameters.isParameter("InputAdapterParameters"))
{
if(rank == 0) std::cerr << "Input adapter parameters not provided" << std::endl;
return;
}
if(!problem_parameters.isParameter("Zoltan2Parameters"))
{
if(rank == 0) std::cerr << "Zoltan2 problem parameters not provided" << std::endl;
return;
}
if(rank == 0)
cout << "\n\nRunning test: " << problem_parameters.name() << endl;
////////////////////////////////////////////////////////////
// 0. add comparison source
////////////////////////////////////////////////////////////
ComparisonSource * comparison_source = new ComparisonSource;
comparison_helper->AddSource(problem_parameters.name(), comparison_source);
comparison_source->addTimer("adapter construction time");
comparison_source->addTimer("problem construction time");
comparison_source->addTimer("solve time");
////////////////////////////////////////////////////////////
// 1. get basic input adapter
////////////////////////////////////////////////////////////
const ParameterList &adapterPlist = problem_parameters.sublist("InputAdapterParameters");
comparison_source->timers["adapter construction time"]->start();
base_adapter_t * ia = AdapterForTests::getAdapterForInput(const_cast<UserInputForTests *>(&uinput), adapterPlist,comm); // a pointer to a basic type
comparison_source->timers["adapter construction time"]->stop();
// if(rank == 0) cout << "Got input adapter... " << endl;
if(ia == nullptr)
{
if(rank == 0)
cout << "Get adapter for input failed" << endl;
return;
}
////////////////////////////////////////////////////////////
// 2. construct a Zoltan2 problem
////////////////////////////////////////////////////////////
string adapter_name = adapterPlist.get<string>("input adapter"); // If we are here we have an input adapter, no need to check for one.
// get Zoltan2 partion parameters
ParameterList zoltan2_parameters = const_cast<ParameterList &>(problem_parameters.sublist("Zoltan2Parameters"));
//if(rank == 0){
// cout << "\nZoltan 2 parameters:" << endl;
// zoltan2_parameters.print(std::cout);
// cout << endl;
//}
comparison_source->timers["problem construction time"]->start();
std::string problem_kind = problem_parameters.get<std::string>("kind");
if (rank == 0) std::cout << "Creating a new " << problem_kind << " problem." << std::endl;
#ifdef HAVE_ZOLTAN2_MPI
base_problem_t * problem =
Zoltan2_TestingFramework::ProblemFactory::newProblem(problem_kind, adapter_name, ia, &zoltan2_parameters, MPI_COMM_WORLD);
#else
base_problem_t * problem =
Zoltan2_TestingFramework::ProblemFactory::newProblem(problem_kind, adapter_name, ia, &zoltan2_parameters);
#endif
if (problem == nullptr) {
if (rank == 0)
std::cerr << "Input adapter type: " + adapter_name + ", is unvailable, or misspelled." << std::endl;
return;
}
////////////////////////////////////////////////////////////
// 3. Solve the problem
////////////////////////////////////////////////////////////
comparison_source->timers["solve time"]->start();
if (problem_kind == "partitioning") {
reinterpret_cast<partitioning_problem_t *>(problem)->solve();
} else if (problem_kind == "ordering") {
reinterpret_cast<ordering_problem_t *>(problem)->solve();
} else if (problem_kind == "coloring") {
reinterpret_cast<coloring_problem_t *>(problem)->solve();
}
comparison_source->timers["solve time"]->stop();
if (rank == 0)
cout << problem_kind + "Problem solved." << endl;
//#define KDDKDD
#ifdef KDDKDD
{
const base_adapter_t::gno_t *kddIDs = NULL;
ia->getIDsView(kddIDs);
for (size_t i = 0; i < ia->getLocalNumIDs(); i++) {
std::cout << rank << " LID " << i
<< " GID " << kddIDs[i]
<< " PART "
<< reinterpret_cast<partitioning_problem_t *>(problem)->getSolution().getPartListView()[i]
<< std::endl;
}
}
#endif
////////////////////////////////////////////////////////////
// 4. Print problem metrics
////////////////////////////////////////////////////////////
// An environment. This is usually created by the problem.
// BDD unused, only applicable to partitioning problems
// RCP<const Zoltan2::Environment> env =
// reinterpret_cast<partitioning_problem_t *>(problem)->getEnvironment();
// get metric object
RCP<EvaluatePartition<basic_id_t> >metricObject =
rcp(Zoltan2_TestingFramework::EvaluatePartitionFactory::
newEvaluatePartition(reinterpret_cast<partitioning_problem_t*>
(problem), adapter_name, ia,
&zoltan2_parameters));
std::string metric_types[] = {"Metrics", "Graph Metrics"};
for (auto metric_type : metric_types) {
if( problem_parameters.isParameter(metric_type)) {
// calculate pass fail based on imbalance
if(rank == 0) cout << "Analyzing " << metric_type << endl;
if(rank == 0) {
std::cout << metric_type << " for " << problem_kind << ":" << std::endl;
if (metric_type == "Metrics")
metricObject->printMetrics(cout);
else if (metric_type == "Graph Metrics")
metricObject->printGraphMetrics(cout);
}
std::ostringstream msg;
auto metricsPlist = problem_parameters.sublist(metric_type); // get the metrics plist
bool all_tests_pass = false;
all_tests_pass
= MetricAnalyzer::analyzeMetrics( metricsPlist,
metricObject,
comm,
msg);
std::cout << msg.str() << std::endl;
if(rank == 0 && all_tests_pass) cout << "All " << metric_type << " tests PASSED." << endl;
else if (rank == 0) cout << "One or more " << metric_type << " tests FAILED." << endl;
} else if (rank == 0) cout << metric_type << " analysis unrequested. PASSED." << endl;
}
#define BDD
#ifdef BDD
if (problem_kind == "ordering") {
std::cout << "\nLet's examine the solution..." << std::endl;
auto solution = reinterpret_cast<ordering_problem_t *>(problem)->getSolution();
if (solution->haveSeparators() ) {
std::ostringstream sol;
sol << "Number of column blocks: " << solution->getNumSeparatorBlocks() << std::endl;
if (solution->getPermutationSize() < 100) {
if (solution->havePerm()) {
sol << "permutation: {";
for (auto &x : solution->getPermutationRCPConst(false)) sol << " " << x;
sol << "}" << std::endl;
}
if (solution->haveInverse()) {
sol << "inverse permutation: {";
for (auto &x : solution->getPermutationRCPConst(true)) sol << " " << x;
sol << "}" << std::endl;
}
if (solution->haveSeparatorRange()) {
sol << "separator range: {";
for (auto &x : solution->getSeparatorRangeRCPConst()) sol << " " << x;
sol << "}" << std::endl;
}
if (solution->haveSeparatorTree()) {
sol << "separator tree: {";
for (auto &x : solution->getSeparatorTreeRCPConst()) sol << " " << x;
sol << "}" << std::endl;
}
}
std::cout << sol.str() << std::endl;
}
}
#endif
// 4b. timers
// if(zoltan2_parameters.isParameter("timer_output_stream"))
// reinterpret_cast<partitioning_problem_t *>(problem)->printTimers();
////////////////////////////////////////////////////////////
// 5. Add solution to map for possible comparison testing
////////////////////////////////////////////////////////////
comparison_source->adapter = RCP<basic_id_t>(reinterpret_cast<basic_id_t *>(ia));
comparison_source->problem = RCP<base_problem_t>(reinterpret_cast<base_problem_t *>(problem));
comparison_source->metricObject = metricObject;
comparison_source->problem_kind = problem_parameters.isParameter("kind") ? problem_parameters.get<string>("kind") : "?";
comparison_source->adapter_kind = adapter_name;
// write mesh solution
// auto sol = reinterpret_cast<partitioning_problem_t *>(problem)->getSolution();
// MyUtils::writePartionSolution(sol.getPartListView(), ia->getLocalNumIDs(), comm);
////////////////////////////////////////////////////////////
// 6. Clean up
////////////////////////////////////////////////////////////
}
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::CrsMatrix <Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inA,
Teuchos::ParameterList& paramListIn,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> >& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
bool hasParamList = paramListIn.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = paramListIn;
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(inA);
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> > &inA,
Teuchos::ParameterList& inParamList,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node>>& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>>& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
typedef Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> crs_matrix_type;
typedef Tpetra::Experimental::BlockCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> block_crs_matrix_type;
bool hasParamList = inParamList.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = inParamList;
RCP<const crs_matrix_type> constCrsA;
RCP<crs_matrix_type> crsA;
#if defined(HAVE_MUELU_EXPERIMENTAL) and defined(HAVE_MUELU_AMGX)
std::string externalMG = "use external multigrid package";
if (hasParamList && paramList.isParameter(externalMG) && paramList.get<std::string>(externalMG) == "amgx"){
constCrsA = rcp_dynamic_cast<const crs_matrix_type>(inA);
TEUCHOS_TEST_FOR_EXCEPTION(constCrsA == Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: failed to dynamic cast to Tpetra::CrsMatrix, which is required to be able to use AmgX.");
return rcp(new AMGXOperator<SC,LO,GO,NO>(inA,inParamList));
}
#endif
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList,inA->getDomainMap()->getComm()));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A;
RCP<block_crs_matrix_type> bcrsA = rcp_dynamic_cast<block_crs_matrix_type>(inA);
crsA = rcp_dynamic_cast<crs_matrix_type>(inA);
if (crsA != Teuchos::null)
A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(crsA);
else if (bcrsA != Teuchos::null) {
RCP<Xpetra::CrsMatrix<SC,LO,GO,NO> > temp = rcp(new Xpetra::TpetraBlockCrsMatrix<SC,LO,GO,NO>(bcrsA));
TEUCHOS_TEST_FOR_EXCEPTION(temp==Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: cast from Tpetra::Experimental::BlockCrsMatrix to Xpetra::TpetraBlockCrsMatrix failed.");
A = rcp(new Xpetra::CrsMatrixWrap<SC,LO,GO,NO>(temp));
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "CreateTpetraPreconditioner: only Tpetra CrsMatrix and BlockCrsMatrix types are supported.");
}
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
using Teuchos::ParameterList;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100, nz = 100;
Galeri::Xpetra::Parameters<GO> galeriParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file [default = 'scalingTest.xml']");
bool printTimings = true; clp.setOption("timings", "notimings", &printTimings, "print timings to screen");
int writeMatricesOPT = -2; clp.setOption("write", &writeMatricesOPT, "write matrices to file (-1 means all; i>=0 means level i)");
std::string dsolveType = "cg", solveType; clp.setOption("solver", &dsolveType, "solve type: (none | cg | gmres | standalone)");
double dtol = 1e-12, tol; clp.setOption("tol", &dtol, "solver convergence tolerance");
std::string mapFile; clp.setOption("map", &mapFile, "map data file");
std::string matrixFile; clp.setOption("matrix", &matrixFile, "matrix data file");
std::string coordFile; clp.setOption("coords", &coordFile, "coordinates data file");
int numRebuilds = 0; clp.setOption("rebuild", &numRebuilds, "#times to rebuild hierarchy");
int maxIts = 200; clp.setOption("its", &maxIts, "maximum number of solver iterations");
bool scaleResidualHistory = true; clp.setOption("scale", "noscale", &scaleResidualHistory, "scaled Krylov residual history");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<ParameterList>(¶mList), *comm);
bool isDriver = paramList.isSublist("Run1");
if (isDriver) {
// update galeriParameters with the values from the XML file
ParameterList& realParams = galeriParameters.GetParameterList();
for (ParameterList::ConstIterator it = realParams.begin(); it != realParams.end(); it++) {
const std::string& name = realParams.name(it);
if (paramList.isParameter(name))
realParams.setEntry(name, paramList.getEntry(name));
}
}
// Retrieve matrix parameters (they may have been changed on the command line)
// [for instance, if we changed matrix type from 2D to 3D we need to update nz]
ParameterList galeriList = galeriParameters.GetParameterList();
// =========================================================================
// Problem construction
// =========================================================================
std::ostringstream galeriStream;
comm->barrier();
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
RCP<TimeMonitor> tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<Matrix> A;
RCP<const Map> map;
RCP<MultiVector> coordinates;
RCP<MultiVector> nullspace;
if (matrixFile.empty()) {
galeriStream << "========================================================\n" << xpetraParameters << galeriParameters;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
std::string matrixType = galeriParameters.GetMatrixType();
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriList);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriList);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriList);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
galeriStream << "Processor subdomains in x direction: " << galeriList.get<int>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriList.get<int>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriList.get<int>("mz") << std::endl
<< "========================================================" << std::endl;
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") {
// Our default test case for elasticity: all boundaries of a square/cube have Neumann b.c. except left which has Dirichlet
galeriList.set("right boundary" , "Neumann");
galeriList.set("bottom boundary", "Neumann");
galeriList.set("top boundary" , "Neumann");
galeriList.set("front boundary" , "Neumann");
galeriList.set("back boundary" , "Neumann");
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(galeriParameters.GetMatrixType(), map, galeriList);
A = Pr->BuildMatrix();
nullspace = MultiVectorFactory::Build(map, 1);
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((galeriParameters.GetMatrixType() == "Elasticity2D") ? 2 : 3);
} else {
nullspace->putScalar(one);
}
} else {
if (!mapFile.empty())
map = Utils2::ReadMap(mapFile, xpetraParameters.GetLib(), comm);
comm->barrier();
if (lib == Xpetra::UseEpetra) {
A = Utils::Read(matrixFile, map);
} else {
// Tpetra matrix reader is still broken, so instead we read in
// a matrix in a binary format and then redistribute it
const bool binaryFormat = true;
A = Utils::Read(matrixFile, lib, comm, binaryFormat);
RCP<Matrix> newMatrix = MatrixFactory::Build(map, 1);
RCP<Import> importer = ImportFactory::Build(A->getRowMap(), map);
newMatrix->doImport(*A, *importer, Xpetra::INSERT);
newMatrix->fillComplete();
A.swap(newMatrix);
}
comm->barrier();
if (!coordFile.empty())
coordinates = Utils2::ReadMultiVector(coordFile, map);
nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(one);
}
comm->barrier();
tm = Teuchos::null;
galeriStream << "Galeri complete.\n========================================================" << std::endl;
int numReruns = 1;
if (paramList.isParameter("number of reruns"))
numReruns = paramList.get<int>("number of reruns");
const bool mustAlreadyExist = true;
for (int rerunCount = 1; rerunCount <= numReruns; rerunCount++) {
ParameterList mueluList, runList;
bool stop = false;
if (isDriver) {
runList = paramList.sublist("Run1", mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} else {
mueluList = paramList;
stop = true;
}
int runCount = 1;
do {
A->SetMaxEigenvalueEstimate(-one);
solveType = dsolveType;
tol = dtol;
int savedOut = -1;
FILE* openedOut = NULL;
if (isDriver) {
if (runList.isParameter("filename")) {
// Redirect all output into a filename We have to redirect all output,
// including printf's, therefore we cannot simply replace C++ cout
// buffers, and have to use heavy machinary (dup2)
std::string filename = runList.get<std::string>("filename");
if (numReruns > 1)
filename += "_run" + MueLu::toString(rerunCount);
filename += (lib == Xpetra::UseEpetra ? ".epetra" : ".tpetra");
savedOut = dup(STDOUT_FILENO);
openedOut = fopen(filename.c_str(), "w");
dup2(fileno(openedOut), STDOUT_FILENO);
}
if (runList.isParameter("solver")) solveType = runList.get<std::string>("solver");
if (runList.isParameter("tol")) tol = runList.get<double> ("tol");
}
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
fancyout << galeriStream.str();
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
RCP<HierarchyManager> mueLuFactory = rcp(new ParameterListInterpreter(mueluList));
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
RCP<Hierarchy> H;
for (int i = 0; i <= numRebuilds; i++) {
A->SetMaxEigenvalueEstimate(-one);
H = mueLuFactory->CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
if (!coordinates.is_null())
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory->SetupHierarchy(*H);
}
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
{
// we set seed for reproducibility
Utils::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
tm = Teuchos::null;
if (writeMatricesOPT > -2) {
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3.5 - Matrix output")));
H->Write(writeMatricesOPT, writeMatricesOPT);
tm = Teuchos::null;
}
comm->barrier();
if (solveType == "none") {
// Do not perform a solve
} else if (solveType == "standalone") {
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Fixed Point Solve")));
H->IsPreconditioner(false);
H->Iterate(*B, *X, maxIts);
} else if (solveType == "cg" || solveType == "gmres") {
#ifdef HAVE_MUELU_BELOS
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp <SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
fancyout << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
if (!scaleResidualHistory)
belosList.set("Implicit Residual Scaling", "None");
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg") {
solver = rcp(new Belos::PseudoBlockCGSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
} else if (solveType == "gmres") {
solver = rcp(new Belos::BlockGmresSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
}
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
fancyout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
} catch(...) {
fancyout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged)
fancyout << std::endl << "ERROR: Belos did not converge! " << std::endl;
else
fancyout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif //ifdef HAVE_MUELU_BELOS
} else {
throw MueLu::Exceptions::RuntimeError("Unknown solver type: \"" + solveType + "\"");
}
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union);
TimeMonitor::clearCounters();
if (isDriver) {
if (openedOut != NULL) {
dup2(savedOut, STDOUT_FILENO);
fclose(openedOut);
openedOut = NULL;
}
try {
runList = paramList.sublist("Run" + MueLu::toString(++runCount), mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} catch (std::exception) {
stop = true;
}
}
} while (stop == false);
}
return 0;
} //main
void MueLuPreconditionerFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node>::
initializePrec(const RCP<const LinearOpSourceBase<Scalar> >& fwdOpSrc, PreconditionerBase<Scalar>* prec, const ESupportSolveUse supportSolveUse) const {
using Teuchos::rcp_dynamic_cast;
// we are using typedefs here, since we are using objects from different packages (Xpetra, Thyra,...)
typedef Xpetra::Map<LocalOrdinal,GlobalOrdinal,Node> XpMap;
typedef Xpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> XpOp;
typedef Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpThyUtils;
typedef Xpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpCrsMat;
typedef Xpetra::BlockedCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpBlockedCrsMat;
typedef Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpMat;
typedef Xpetra::MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> XpMultVec;
typedef Xpetra::MultiVector<double,LocalOrdinal,GlobalOrdinal,Node> XpMultVecDouble;
typedef Thyra::LinearOpBase<Scalar> ThyLinOpBase;
#ifdef HAVE_MUELU_TPETRA
typedef MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> MueTpOp;
typedef Tpetra::Operator<Scalar,LocalOrdinal,GlobalOrdinal,Node> TpOp;
typedef Thyra::TpetraLinearOp<Scalar,LocalOrdinal,GlobalOrdinal,Node> ThyTpLinOp;
#endif
// Check precondition
TEUCHOS_ASSERT(Teuchos::nonnull(fwdOpSrc));
TEUCHOS_ASSERT(this->isCompatible(*fwdOpSrc));
TEUCHOS_ASSERT(prec);
// Create a copy, as we may remove some things from the list
ParameterList paramList = *paramList_;
// Retrieve wrapped concrete Xpetra matrix from FwdOp
const RCP<const ThyLinOpBase> fwdOp = fwdOpSrc->getOp();
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(fwdOp));
// Check whether it is Epetra/Tpetra
bool bIsEpetra = XpThyUtils::isEpetra(fwdOp);
bool bIsTpetra = XpThyUtils::isTpetra(fwdOp);
bool bIsBlocked = XpThyUtils::isBlockedOperator(fwdOp);
TEUCHOS_TEST_FOR_EXCEPT((bIsEpetra == true && bIsTpetra == true));
TEUCHOS_TEST_FOR_EXCEPT((bIsEpetra == bIsTpetra) && bIsBlocked == false);
TEUCHOS_TEST_FOR_EXCEPT((bIsEpetra != bIsTpetra) && bIsBlocked == true);
RCP<XpMat> A = Teuchos::null;
if(bIsBlocked) {
Teuchos::RCP<const Thyra::BlockedLinearOpBase<Scalar> > ThyBlockedOp =
Teuchos::rcp_dynamic_cast<const Thyra::BlockedLinearOpBase<Scalar> >(fwdOp);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(ThyBlockedOp));
TEUCHOS_TEST_FOR_EXCEPT(ThyBlockedOp->blockExists(0,0)==false);
Teuchos::RCP<const LinearOpBase<Scalar> > b00 = ThyBlockedOp->getBlock(0,0);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(b00));
RCP<const XpCrsMat > xpetraFwdCrsMat00 = XpThyUtils::toXpetra(b00);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMat00));
// MueLu needs a non-const object as input
RCP<XpCrsMat> xpetraFwdCrsMatNonConst00 = Teuchos::rcp_const_cast<XpCrsMat>(xpetraFwdCrsMat00);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMatNonConst00));
// wrap the forward operator as an Xpetra::Matrix that MueLu can work with
RCP<XpMat> A00 = rcp(new Xpetra::CrsMatrixWrap<Scalar,LocalOrdinal,GlobalOrdinal,Node>(xpetraFwdCrsMatNonConst00));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(A00));
RCP<const XpMap> rowmap00 = A00->getRowMap();
RCP< const Teuchos::Comm< int > > comm = rowmap00->getComm();
// create a Xpetra::BlockedCrsMatrix which derives from Xpetra::Matrix that MueLu can work with
RCP<XpBlockedCrsMat> bMat = Teuchos::rcp(new XpBlockedCrsMat(ThyBlockedOp, comm));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(bMat));
// save blocked matrix
A = bMat;
} else {
RCP<const XpCrsMat > xpetraFwdCrsMat = XpThyUtils::toXpetra(fwdOp);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMat));
// MueLu needs a non-const object as input
RCP<XpCrsMat> xpetraFwdCrsMatNonConst = Teuchos::rcp_const_cast<XpCrsMat>(xpetraFwdCrsMat);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(xpetraFwdCrsMatNonConst));
// wrap the forward operator as an Xpetra::Matrix that MueLu can work with
A = rcp(new Xpetra::CrsMatrixWrap<Scalar,LocalOrdinal,GlobalOrdinal,Node>(xpetraFwdCrsMatNonConst));
}
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(A));
// Retrieve concrete preconditioner object
const Teuchos::Ptr<DefaultPreconditioner<Scalar> > defaultPrec = Teuchos::ptr(dynamic_cast<DefaultPreconditioner<Scalar> *>(prec));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(defaultPrec));
// extract preconditioner operator
RCP<ThyLinOpBase> thyra_precOp = Teuchos::null;
thyra_precOp = rcp_dynamic_cast<Thyra::LinearOpBase<Scalar> >(defaultPrec->getNonconstUnspecifiedPrecOp(), true);
// Variable for multigrid hierarchy: either build a new one or reuse the existing hierarchy
RCP<MueLu::Hierarchy<Scalar,LocalOrdinal,GlobalOrdinal,Node> > H = Teuchos::null;
// make a decision whether to (re)build the multigrid preconditioner or reuse the old one
// rebuild preconditioner if startingOver == true
// reuse preconditioner if startingOver == false
const bool startingOver = (thyra_precOp.is_null() || !paramList.isParameter("reuse: type") || paramList.get<std::string>("reuse: type") == "none");
if (startingOver == true) {
// extract coordinates from parameter list
Teuchos::RCP<XpMultVecDouble> coordinates = Teuchos::null;
coordinates = MueLu::Utilities<Scalar,LocalOrdinal,GlobalOrdinal,Node>::ExtractCoordinatesFromParameterList(paramList);
// TODO check for Xpetra or Thyra vectors?
RCP<XpMultVec> nullspace = Teuchos::null;
#ifdef HAVE_MUELU_TPETRA
if (bIsTpetra) {
typedef Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> tMV;
RCP<tMV> tpetra_nullspace = Teuchos::null;
if (paramList.isType<Teuchos::RCP<tMV> >("Nullspace")) {
tpetra_nullspace = paramList.get<RCP<tMV> >("Nullspace");
paramList.remove("Nullspace");
nullspace = MueLu::TpetraMultiVector_To_XpetraMultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node>(tpetra_nullspace);
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(nullspace));
}
}
#endif
// build a new MueLu hierarchy
H = MueLu::CreateXpetraPreconditioner(A, paramList, coordinates, nullspace);
} else {
// reuse old MueLu hierarchy stored in MueLu Tpetra/Epetra operator and put in new matrix
// get old MueLu hierarchy
#if defined(HAVE_MUELU_TPETRA)
if (bIsTpetra) {
RCP<ThyTpLinOp> tpetr_precOp = rcp_dynamic_cast<ThyTpLinOp>(thyra_precOp);
RCP<MueTpOp> muelu_precOp = rcp_dynamic_cast<MueTpOp>(tpetr_precOp->getTpetraOperator(),true);
H = muelu_precOp->GetHierarchy();
}
#endif
// TODO add the blocked matrix case here...
TEUCHOS_TEST_FOR_EXCEPTION(!H->GetNumLevels(), MueLu::Exceptions::RuntimeError,
"Thyra::MueLuPreconditionerFactory: Hierarchy has no levels in it");
TEUCHOS_TEST_FOR_EXCEPTION(!H->GetLevel(0)->IsAvailable("A"), MueLu::Exceptions::RuntimeError,
"Thyra::MueLuPreconditionerFactory: Hierarchy has no fine level operator");
RCP<MueLu::Level> level0 = H->GetLevel(0);
RCP<XpOp> O0 = level0->Get<RCP<XpOp> >("A");
RCP<XpMat> A0 = rcp_dynamic_cast<XpMat>(O0);
if (!A0.is_null()) {
// If a user provided a "number of equations" argument in a parameter list
// during the initial setup, we must honor that settings and reuse it for
// all consequent setups.
A->SetFixedBlockSize(A0->GetFixedBlockSize());
}
// set new matrix
level0->Set("A", A);
H->SetupRe();
}
// wrap hierarchy H in thyraPrecOp
RCP<ThyLinOpBase > thyraPrecOp = Teuchos::null;
#if defined(HAVE_MUELU_TPETRA)
if (bIsTpetra) {
RCP<MueTpOp> muelu_tpetraOp = rcp(new MueTpOp(H));
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(muelu_tpetraOp));
RCP<TpOp> tpOp = Teuchos::rcp_dynamic_cast<TpOp>(muelu_tpetraOp);
thyraPrecOp = Thyra::createLinearOp<Scalar, LocalOrdinal, GlobalOrdinal, Node>(tpOp);
}
#endif
if(bIsBlocked) {
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::nonnull(thyraPrecOp));
typedef MueLu::XpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> MueXpOp;
//typedef Thyra::XpetraLinearOp<Scalar,LocalOrdinal,GlobalOrdinal,Node> ThyXpLinOp; // unused
const RCP<MueXpOp> muelu_xpetraOp = rcp(new MueXpOp(H));
RCP<const VectorSpaceBase<Scalar> > thyraRangeSpace = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyra(muelu_xpetraOp->getRangeMap());
RCP<const VectorSpaceBase<Scalar> > thyraDomainSpace = Xpetra::ThyraUtils<Scalar,LocalOrdinal,GlobalOrdinal,Node>::toThyra(muelu_xpetraOp->getDomainMap());
RCP <Xpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> > xpOp = Teuchos::rcp_dynamic_cast<Xpetra::Operator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >(muelu_xpetraOp);
thyraPrecOp = Thyra::xpetraLinearOp(thyraRangeSpace, thyraDomainSpace,xpOp);
}
TEUCHOS_TEST_FOR_EXCEPT(Teuchos::is_null(thyraPrecOp));
defaultPrec->initializeUnspecified(thyraPrecOp);
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
bool success = false;
bool verbose = true;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int numProc = comm->getSize();
int myRank = comm->getRank();
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& out = *fancy;
out.setOutputToRootOnly(0);
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
Xpetra::Parameters xpetraParameters(clp);
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
const int numLists = 1;
std::vector<std::string> dirList;
dirList.push_back("Convergence/Laplace2D/");
bool failed = false;
for (int k = 0; k < numLists; k++) {
const std::string& dirName = dirList[k];
std::string problemFile = dirName + "problem.xml";
ParameterList galeriParameters;
Teuchos::updateParametersFromXmlFileAndBroadcast(problemFile, Teuchos::Ptr<Teuchos::ParameterList>(&galeriParameters), *comm);
if (!galeriParameters.isParameter("mz"))
galeriParameters.set<int>("mz", -1);
// =========================================================================
// Problem construction (copy-paste from Driver.cpp)
// =========================================================================
RCP<Matrix> A;
RCP<Map> map;
RCP<MultiVector> nullspace, coordinates;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriParameters.
std::string matrixType = galeriParameters.get<std::string>("matrixType");
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian1D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriParameters);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian2D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriParameters);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian3D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriParameters);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
#if 0
out << "========================================================\n" << xpetraParameters << galeriParameters;
out << "Processor subdomains in x direction: " << galeriParameters.get<GO>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriParameters.get<GO>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriParameters.get<GO>("mz") << std::endl
<< "========================================================" << std::endl;
#endif
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixType, map, galeriParameters);
A = Pr->BuildMatrix();
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((matrixType == "Elasticity2D") ? 2 : 3);
} else {
nullspace = MultiVectorFactory::Build(map, 1);
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(0);
for (int i = 0; i < nsData.size(); i++)
nsData[i] = one;
}
// =========================================================================
// Run different configurations
// =========================================================================
Teuchos::ArrayRCP<std::string> fileList = MueLuTests::TestHelpers::GetFileList(dirList[k],
(numProc == 1 ? std::string(".xml") : std::string("_np" + Teuchos::toString(numProc) + ".xml")));
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
for (int i = 0; i < fileList.size(); i++) {
if (fileList[i] == "problem.xml")
continue;
// Set seed
Utilities::SetRandomSeed(*comm);
// Reset (potentially) cached value of the estimate
A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());
std::string xmlFile = dirName + fileList[i];
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFile, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
std::string solveType = paramList.get<std::string> ("solver", "standalone");
double goldRate = paramList.get<double> ("convergence rate");
ParameterList& mueluList = paramList.sublist ("MueLu");
TEUCHOS_TEST_FOR_EXCEPTION(solveType != "standalone" && solveType != "cg" && solveType != "gmres", MueLu::Exceptions::RuntimeError,
"Unknown solver type \"" << solveType << "\"");
bool isPrec = !(solveType == "standalone");
if (!mueluList.isParameter("verbosity"))
mueluList.set("verbosity", "none");
#ifndef HAVE_MUELU_BELOS
if (isPrec)
out << xmlFile << ": skipped (Belos is not enabled)" << std::endl;
#endif
// =========================================================================
// Preconditioner construction
// =========================================================================
RCP<Hierarchy> H;
try {
ParameterListInterpreter mueluFactory(mueluList);
H = mueluFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("Coordinates", coordinates);
mueluFactory.SetupHierarchy(*H);
} catch (Teuchos::ExceptionBase& e) {
std::string msg = e.what();
msg = msg.substr(msg.find_last_of('\n')+1);
out << "Caught exception: " << msg << std::endl;
if (msg == "Zoltan interface is not available" ||
msg == "Zoltan2 interface is not available") {
if (myRank == 0)
out << xmlFile << ": skipped (missing library)" << std::endl;
continue;
}
}
// Set X, B
{
// TODO: do multiple vectors simultaneously to average
// we set seed for reproducibility
Utilities::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
const int maxIts = 100;
const double tol = 1e-12;
H->IsPreconditioner(isPrec);
if (isPrec == false) {
MueLu::ReturnType ret = H->Iterate(*B, *X, std::pair<LO,SC>(maxIts, tol));
double rate = H->GetRate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} else {
#ifdef HAVE_MUELU_BELOS
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
// Define Operator and Preconditioner
RCP<OP> belosOp = rcp(new Belos::XpetraOp<SC, LO, GO, NO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
RCP<OP> belosPrec = rcp(new Belos::MueLuOp <SC, LO, GO, NO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
out << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
#if 1
belosList.set("Verbosity", Belos::Errors + Belos::Warnings);
#else
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
#endif
// Belos custom test to store residuals
// Create an iterative solver manager
RCP<Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg")
solver = rcp(new Belos::PseudoBlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
else if (solveType == "gmres")
solver = rcp(new Belos::BlockGmresSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
RCP<Belos::MyStatusTest<SC, MV, OP> > status = rcp(new Belos::MyStatusTest<SC, MV, OP>(tol));
solver->setDebugStatusTest(status);
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
double rate = status->rate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} catch(...) {
out << xmlFile << ": failed (exception)" << std::endl;
failed = true;
}
#endif //ifdef HAVE_MUELU_BELOS
}
}
}
success = !failed;
out << std::endl << "End Result: TEST " << (failed ? "FAILED" : "PASSED") << std::endl;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
static MetricAnalyzerInfo getMetricInfo( const ParameterList & metricCheckParameters,
const RCP<const Zoltan2::EvaluatePartition <basic_id_t> > &metricObject) {
MetricAnalyzerInfo result; // will fill these values
for (auto iterateAllKeys = metricCheckParameters.begin(); iterateAllKeys != metricCheckParameters.end(); ++iterateAllKeys) {
auto checkName = metricCheckParameters.name(iterateAllKeys);
if ( checkName != WEIGHT_PARAMETER_NAME &&
checkName != KEYWORD_PARAMETER_NAME &&
checkName != UPPER_PARAMETER_NAME &&
checkName != LOWER_PARAMETER_NAME &&
checkName != NORMED_PARAMETER_NAME ) {
throw std::logic_error( "Key name: '" + checkName + "' is not understood." );
}
}
// pick up the weight index - this parameter is optional so we check it first - this way we can communicate with EvaluatePartition properly in the next step
int selectedWeightIndex = UNDEFINED_PARAMETER_INT_INDEX; // meaning not specified so default case
if( metricCheckParameters.isParameter(WEIGHT_PARAMETER_NAME)) {
selectedWeightIndex = metricCheckParameters.get<int>(WEIGHT_PARAMETER_NAME);
if( selectedWeightIndex < 0 ) {
throw std::logic_error( "Optional weight index was specified as: " + std::to_string(selectedWeightIndex) + " Weight index must be 0 or positive." ); // I think that's the best I can do for error checking weight index right now - we may want to specify the cap when we know it
}
}
// pick up the norm index - this parameter is optional so we check it first - this way we can communicate with EvaluatePartition properly in the next step
int selectedNormedSetting = UNDEFINED_PARAMETER_INT_INDEX; // meaning not specified so default case
if( metricCheckParameters.isParameter(NORMED_PARAMETER_NAME)) {
bool bNormSetting = metricCheckParameters.get<bool>(NORMED_PARAMETER_NAME);
selectedNormedSetting = bNormSetting ? 1 : 0;
if( selectedNormedSetting != 0 && selectedNormedSetting != 1 ) {
throw std::logic_error( "Optional normed parameter was specified as: " + std::to_string(selectedNormedSetting) + " Normed parameter must be true or false." );
}
}
// one of the parameters called "check" should define a string which is a keyword which correlates to an EvaluatePartition API all
// this area needs some consideration - how and where shall we map and define the naming conventions
if( metricCheckParameters.isParameter(KEYWORD_PARAMETER_NAME)) {
std::string theKeyWord = metricCheckParameters.get<std::string>(KEYWORD_PARAMETER_NAME);
// this is going to need some consideration - how is the adapter scalar_t type to be properly handled?
result.theValue = convertParameterChoicesToEvaluatePartitionAPICall(metricObject, theKeyWord, selectedWeightIndex, selectedNormedSetting );
// now we can obtain the upper and lower bounds for this test
result.bFoundUpperBound = metricCheckParameters.isParameter(UPPER_PARAMETER_NAME);
result.bFoundLowerBound = metricCheckParameters.isParameter(LOWER_PARAMETER_NAME);
if (!result.bFoundUpperBound && !result.bFoundLowerBound) {
throw std::logic_error( "The parameter list failed to find an entry for '" + std::string(UPPER_PARAMETER_NAME) + "' or '" + std::string(LOWER_PARAMETER_NAME) + "' and at least one is required." );
}
else if (result.bFoundUpperBound && result.bFoundLowerBound) {
result.lowerValue = metricCheckParameters.get<double>(LOWER_PARAMETER_NAME);
result.upperValue = metricCheckParameters.get<double>(UPPER_PARAMETER_NAME);
}
else if (result.bFoundUpperBound) {
result.upperValue = metricCheckParameters.get<double>(UPPER_PARAMETER_NAME);
}
else {
result.lowerValue = metricCheckParameters.get<double>(LOWER_PARAMETER_NAME);
}
result.parameterDescription = theKeyWord;
if( selectedWeightIndex != UNDEFINED_PARAMETER_INT_INDEX ) {
result.parameterDescription = result.parameterDescription + " (weight: " + std::to_string(selectedWeightIndex) + ")";
}
else if( selectedNormedSetting != UNDEFINED_PARAMETER_INT_INDEX ) { // throw above would catch the case where both of these were set
result.parameterDescription = result.parameterDescription + " (normed: " + ( ( selectedNormedSetting == 0 ) ? "false" : "true" ) + ")";
}
}
return result;
}
