static bool analyzePartitionMetrics( const ParameterList &metricsPlist,
const RCP<const Zoltan2::EvaluatePartition <basic_id_t> > &metricObject,
const RCP<const Comm<int>> &comm,
std::ostringstream &msg_stream) {
ArrayRCP<const metric_t> metrics = metricObject->getMetrics();
bool all_tests_pass = true;
zscalar_t metric_value = 0.0;
for (int i = 0; i < metrics.size(); i++) {
// print their names...
if (metricsPlist.isSublist(metrics[i].getName())) {
auto metric_plist = metricsPlist.sublist(metrics[i].getName());
// loop on tests
auto p= metric_plist.begin(); // iterator
while (p != metric_plist.end()) {
auto test_name = metric_plist.name(p);
if( metrics[i].hasMetricValue(test_name)) {
if(!MetricAnalyzer::MetricBoundsTest( metrics[i].getMetricValue(test_name),
test_name,
metric_plist.sublist(test_name),
comm,
msg_stream)) {
all_tests_pass = false;
}
} else msg_stream << "UNKNOWN TEST: " + test_name << std::endl;
++p;
}
} else {
msg_stream << "UNKNOWN METRIC: " + metrics[i].getName() << std::endl;
}
}
return all_tests_pass;
}
NonlinearSolver<double> NonlinearSolverBuilder::createSolver(const ParameterList& params)
{
if (params.isSublist("NOX Solver"))
{
return new NOXSolver(params);
}
else if (params.isSublist("Nonlinear Solver"))
{
ParameterList sub = params.sublist("Nonlinear Solver");
Array<string> names = tuple<string>("Newton Armijo Solver", "Newton-Armijo Solver", "NewtonArmijoSolver");
for (int i=0; i<names.size(); i++)
{
if (sub.isSublist(names[i]))
{
ParameterList subsub = sub.sublist(names[i]);
LinearSolver<double> linSolver;
if (subsub.isParameter("Linear Solver"))
{
string solverFile = subsub.get<string>("Linear Solver");
linSolver = LinearSolverBuilder::createSolver(solverFile);
}
else if (subsub.isSublist("Linear Solver"))
{
linSolver = LinearSolverBuilder::createSolver(subsub);
}
else
{
TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
"Nonlinear solver parameter list " << sub
<< " does not appear to specify a solver for the linear subproblems");
}
return new NewtonArmijoSolver<double>(subsub, linSolver);
}
}
}
else
{
TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
"Nonlinear solver parameter list " << params
<< " can't be parsed to find a nonlinear solver");
}
return NonlinearSolver<double>();
}
BelosSolver::BelosSolver(const ParameterList& params)
: LinearSolverBase<double>(params), pf_(), hasSolver_(false)
{
if (params.isSublist("Preconditioner"))
{
ParameterList precParams = params.sublist("Preconditioner");
pf_ = new ParameterListPreconditionerFactory(precParams);
}
}
std::queue<ParameterList> ComparisonHelper::getMetricsToCompare(const ParameterList &pList)
{
// extract all of the metrics to be tested
std::queue<ParameterList> metrics;
for(auto it = pList.begin(); it != pList.end(); ++it) {
if (pList.isSublist(it->first)) {
metrics.push(pList.sublist(it->first));
}
}
return metrics;
}
PCDPreconditionerFactory::PCDPreconditionerFactory(
const ParameterList& params,
const LinearProblem& MpProb,
const LinearProblem& ApProb,
const LinearProblem& FpProb
)
: MpProb_(MpProb),
ApProb_(ApProb),
FpProb_(FpProb),
MpSolver_(),
ApSolver_(),
FSolver_()
{
ParameterList msParams = params.sublist("MpSolver");
MpSolver_ = LinearSolverBuilder::createSolver(msParams);
ParameterList asParams = params.sublist("ApSolver");
ApSolver_ = LinearSolverBuilder::createSolver(asParams);
ParameterList fsParams = params.sublist("FSolver");
FSolver_ = LinearSolverBuilder::createSolver(fsParams);
}
static void LoadMetricInfo(std::vector<MetricAnalyzerInfo> & metricInfoSet,
const RCP<const Zoltan2::EvaluatePartition <basic_id_t> > &metricObject,
const ParameterList &metricsParameters) {
// at this point we should be looking at a metricsPlist with the following format - note that weight is optional
// <ParameterList name="metriccheck1">
// <Parameter name="check" type="string" value="imbalance"/>
// <Parameter name="lower" type="double" value="0.99"/>
// <Parameter name="upper" type="double" value="1.4"/>
// </ParameterList>
// <ParameterList name="metriccheck2">
// <Parameter name="check" type="string" value="imbalance"/>
// <Parameter name="weight" type="int" value="0"/>
// <Parameter name="lower" type="double" value="0.99"/>
// <Parameter name="upper" type="double" value="1.4"/>
// </ParameterList>
// first let's get a list of all the headings, so "metriccheck1", "metriccheck2" in this case
// I've currently got this enforcing those names strictly to make sure formatting is correct
// But really the headings could just be any unique names and are arbitrary
int headingIndex = 1;
for (auto iterateArbitraryHeadingNames = metricsParameters.begin(); iterateArbitraryHeadingNames != metricsParameters.end(); ++iterateArbitraryHeadingNames) {
auto headingName = metricsParameters.name(iterateArbitraryHeadingNames);
// we could be flexible on these headers but for now let's enforce it to get any convention inconsistencies cleaned up
std::string expectedHeadingName = "metriccheck" + std::to_string(headingIndex);
if( expectedHeadingName != headingName) {
throw std::logic_error( "The parameter list expected to find a heading with name '" + expectedHeadingName + "' but instead found '" + headingName );
}
// get the parameters specific to the check we want to run
const ParameterList & metricCheckParameters = metricsParameters.sublist(headingName);
MetricAnalyzerInfo metricInfo = getMetricInfo(metricCheckParameters, metricObject);
metricInfoSet.push_back(metricInfo);
++headingIndex;
}
}
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::CrsMatrix <Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inA,
Teuchos::ParameterList& paramListIn,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> >& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
bool hasParamList = paramListIn.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = paramListIn;
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(inA);
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
int
main (int argc, char *argv[])
{
using namespace TrilinosCouplings; // Yes, this means I'm lazy.
using TpetraIntrepidPoissonExample::exactResidualNorm;
using TpetraIntrepidPoissonExample::makeMatrixAndRightHandSide;
using TpetraIntrepidPoissonExample::solveWithBelos;
using TpetraIntrepidPoissonExample::solveWithBelosGPU;
using IntrepidPoissonExample::makeMeshInput;
using IntrepidPoissonExample::setCommandLineArgumentDefaults;
using IntrepidPoissonExample::setUpCommandLineArguments;
using IntrepidPoissonExample::parseCommandLineArguments;
using Tpetra::DefaultPlatform;
using Teuchos::Comm;
using Teuchos::outArg;
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcpFromRef;
using Teuchos::getFancyOStream;
using Teuchos::FancyOStream;
using std::endl;
// Pull in typedefs from the example's namespace.
typedef TpetraIntrepidPoissonExample::ST ST;
typedef TpetraIntrepidPoissonExample::LO LO;
typedef TpetraIntrepidPoissonExample::GO GO;
typedef TpetraIntrepidPoissonExample::Node Node;
typedef Teuchos::ScalarTraits<ST> STS;
typedef STS::magnitudeType MT;
typedef Teuchos::ScalarTraits<MT> STM;
typedef TpetraIntrepidPoissonExample::sparse_matrix_type sparse_matrix_type;
typedef TpetraIntrepidPoissonExample::vector_type vector_type;
typedef TpetraIntrepidPoissonExample::operator_type operator_type;
bool success = true;
try {
Teuchos::oblackholestream blackHole;
Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
const int myRank = mpiSession.getRank ();
//const int numProcs = mpiSession.getNProc ();
// Get the default communicator and Kokkos Node instance
RCP<const Comm<int> > comm =
DefaultPlatform::getDefaultPlatform ().getComm ();
RCP<Node> node = DefaultPlatform::getDefaultPlatform ().getNode ();
// Did the user specify --help at the command line to print help
// with command-line arguments?
bool printedHelp = false;
// Values of command-line arguments.
int nx, ny, nz;
std::string xmlInputParamsFile;
bool verbose, debug;
// Set default values of command-line arguments.
setCommandLineArgumentDefaults (nx, ny, nz, xmlInputParamsFile,
verbose, debug);
// Parse and validate command-line arguments.
Teuchos::CommandLineProcessor cmdp (false, true);
setUpCommandLineArguments (cmdp, nx, ny, nz, xmlInputParamsFile,
verbose, debug);
bool gpu = false;
cmdp.setOption ("gpu", "no-gpu", &gpu,
"Run example using GPU node (if supported)");
int ranks_per_node = 1;
cmdp.setOption("ranks_per_node", &ranks_per_node,
"Number of MPI ranks per node");
int gpu_ranks_per_node = 1;
cmdp.setOption("gpu_ranks_per_node", &gpu_ranks_per_node,
"Number of MPI ranks per node for GPUs");
int device_offset = 0;
cmdp.setOption("device_offset", &device_offset,
"Offset for attaching MPI ranks to CUDA devices");
parseCommandLineArguments (cmdp, printedHelp, argc, argv, nx, ny, nz,
xmlInputParamsFile, verbose, debug);
if (printedHelp) {
// The user specified --help at the command line to print help
// with command-line arguments. We printed help already, so quit
// with a happy return code.
return EXIT_SUCCESS;
}
// Both streams only print on MPI Rank 0. "out" only prints if the
// user specified --verbose.
RCP<FancyOStream> out =
getFancyOStream (rcpFromRef ((myRank == 0 && verbose) ? std::cout : blackHole));
RCP<FancyOStream> err =
getFancyOStream (rcpFromRef ((myRank == 0 && debug) ? std::cerr : blackHole));
#ifdef HAVE_MPI
*out << "PARALLEL executable" << endl;
#else
*out << "SERIAL executable" << endl;
#endif
/**********************************************************************************/
/********************************** GET XML INPUTS ********************************/
/**********************************************************************************/
ParameterList inputList;
if (xmlInputParamsFile != "") {
*out << "Reading parameters from XML file \""
<< xmlInputParamsFile << "\"..." << endl;
Teuchos::updateParametersFromXmlFile (xmlInputParamsFile,
outArg (inputList));
if (myRank == 0) {
inputList.print (*out, 2, true, true);
*out << endl;
}
}
// Get Pamgen mesh definition string, either from the input
// ParameterList or from our function that makes a cube and fills in
// the number of cells along each dimension.
std::string meshInput = inputList.get("meshInput", "");
if (meshInput == "") {
*out << "Generating mesh input string: nx = " << nx
<< ", ny = " << ny
<< ", nz = " << nz << endl;
meshInput = makeMeshInput (nx, ny, nz);
}
// Total application run time
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Time", total_time);
RCP<sparse_matrix_type> A;
RCP<vector_type> B, X_exact, X;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Assembly", total_assembly);
makeMatrixAndRightHandSide (A, B, X_exact, X, comm, node, meshInput,
out, err, verbose, debug);
}
const std::vector<MT> norms = exactResidualNorm (A, B, X_exact);
// X_exact is the exact solution of the PDE, projected onto the
// discrete mesh. It may not necessarily equal the exact solution
// of the linear system.
*out << "||B - A*X_exact||_2 = " << norms[0] << endl
<< "||B||_2 = " << norms[1] << endl
<< "||A||_F = " << norms[2] << endl;
// Setup preconditioner
std::string prec_type = inputList.get("Preconditioner", "None");
RCP<operator_type> M;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Preconditioner Setup", total_prec);
if (prec_type == "MueLu") {
if (inputList.isSublist("MueLu")) {
ParameterList mueluParams = inputList.sublist("MueLu");
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A,mueluParams);
} else {
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A);
}
}
}
bool converged = false;
int numItersPerformed = 0;
const MT tol = inputList.get("Convergence Tolerance",
STM::squareroot (STM::eps ()));
const int maxNumIters = inputList.get("Maximum Iterations", 200);
const int num_steps = inputList.get("Number of Time Steps", 1);
if (gpu) {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total GPU Solve", total_solve);
solveWithBelosGPU(converged, numItersPerformed, tol, maxNumIters, num_steps,
ranks_per_node, gpu_ranks_per_node, device_offset,
prec_type,
X, A, B, Teuchos::null, M);
}
else {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Solve", total_solve);
solveWithBelos (converged, numItersPerformed, tol, maxNumIters, num_steps,
X, A, B, Teuchos::null, M);
}
// Compute ||X-X_exact||_2
MT norm_x = X_exact->norm2();
X_exact->update(-1.0, *X, 1.0);
MT norm_error = X_exact->norm2();
*out << endl
<< "||X-X_exact||_2 / ||X_exact||_2 = " << norm_error / norm_x
<< endl;
} // total time block
// Summarize timings
// RCP<ParameterList> reportParams = parameterList ("TimeMonitor::report");
// reportParams->set ("Report format", std::string ("YAML"));
// reportParams->set ("writeGlobalStats", true);
// Teuchos::TimeMonitor::report (*out, reportParams);
Teuchos::TimeMonitor::summarize(std::cout);
} //try
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
if (success)
return EXIT_SUCCESS;
return EXIT_FAILURE;
}
void run(const UserInputForTests &uinput,
const ParameterList &problem_parameters,
RCP<ComparisonHelper> & comparison_helper,
const RCP<const Teuchos::Comm<int> > & comm)
{
// Major steps in running a problem in zoltan 2
// 1. get an input adapter
// 2. construct the problem
// 3. solve the problem
// 4. analyze metrics
// 5. clean up
int rank = comm->getRank();
if(!problem_parameters.isParameter("kind"))
{
if(rank == 0) std::cerr << "Problem kind not provided" << std::endl;
return;
}
if(!problem_parameters.isParameter("InputAdapterParameters"))
{
if(rank == 0) std::cerr << "Input adapter parameters not provided" << std::endl;
return;
}
if(!problem_parameters.isParameter("Zoltan2Parameters"))
{
if(rank == 0) std::cerr << "Zoltan2 problem parameters not provided" << std::endl;
return;
}
if(rank == 0)
cout << "\n\nRunning test: " << problem_parameters.name() << endl;
////////////////////////////////////////////////////////////
// 0. add comparison source
////////////////////////////////////////////////////////////
ComparisonSource * comparison_source = new ComparisonSource;
comparison_helper->AddSource(problem_parameters.name(), comparison_source);
comparison_source->addTimer("adapter construction time");
comparison_source->addTimer("problem construction time");
comparison_source->addTimer("solve time");
////////////////////////////////////////////////////////////
// 1. get basic input adapter
////////////////////////////////////////////////////////////
const ParameterList &adapterPlist = problem_parameters.sublist("InputAdapterParameters");
comparison_source->timers["adapter construction time"]->start();
base_adapter_t * ia = AdapterForTests::getAdapterForInput(const_cast<UserInputForTests *>(&uinput), adapterPlist,comm); // a pointer to a basic type
comparison_source->timers["adapter construction time"]->stop();
// if(rank == 0) cout << "Got input adapter... " << endl;
if(ia == nullptr)
{
if(rank == 0)
cout << "Get adapter for input failed" << endl;
return;
}
////////////////////////////////////////////////////////////
// 2. construct a Zoltan2 problem
////////////////////////////////////////////////////////////
string adapter_name = adapterPlist.get<string>("input adapter"); // If we are here we have an input adapter, no need to check for one.
// get Zoltan2 partion parameters
ParameterList zoltan2_parameters = const_cast<ParameterList &>(problem_parameters.sublist("Zoltan2Parameters"));
//if(rank == 0){
// cout << "\nZoltan 2 parameters:" << endl;
// zoltan2_parameters.print(std::cout);
// cout << endl;
//}
comparison_source->timers["problem construction time"]->start();
std::string problem_kind = problem_parameters.get<std::string>("kind");
if (rank == 0) std::cout << "Creating a new " << problem_kind << " problem." << std::endl;
#ifdef HAVE_ZOLTAN2_MPI
base_problem_t * problem =
Zoltan2_TestingFramework::ProblemFactory::newProblem(problem_kind, adapter_name, ia, &zoltan2_parameters, MPI_COMM_WORLD);
#else
base_problem_t * problem =
Zoltan2_TestingFramework::ProblemFactory::newProblem(problem_kind, adapter_name, ia, &zoltan2_parameters);
#endif
if (problem == nullptr) {
if (rank == 0)
std::cerr << "Input adapter type: " + adapter_name + ", is unvailable, or misspelled." << std::endl;
return;
}
////////////////////////////////////////////////////////////
// 3. Solve the problem
////////////////////////////////////////////////////////////
comparison_source->timers["solve time"]->start();
if (problem_kind == "partitioning") {
reinterpret_cast<partitioning_problem_t *>(problem)->solve();
} else if (problem_kind == "ordering") {
reinterpret_cast<ordering_problem_t *>(problem)->solve();
} else if (problem_kind == "coloring") {
reinterpret_cast<coloring_problem_t *>(problem)->solve();
}
comparison_source->timers["solve time"]->stop();
if (rank == 0)
cout << problem_kind + "Problem solved." << endl;
//#define KDDKDD
#ifdef KDDKDD
{
const base_adapter_t::gno_t *kddIDs = NULL;
ia->getIDsView(kddIDs);
for (size_t i = 0; i < ia->getLocalNumIDs(); i++) {
std::cout << rank << " LID " << i
<< " GID " << kddIDs[i]
<< " PART "
<< reinterpret_cast<partitioning_problem_t *>(problem)->getSolution().getPartListView()[i]
<< std::endl;
}
}
#endif
////////////////////////////////////////////////////////////
// 4. Print problem metrics
////////////////////////////////////////////////////////////
// An environment. This is usually created by the problem.
// BDD unused, only applicable to partitioning problems
// RCP<const Zoltan2::Environment> env =
// reinterpret_cast<partitioning_problem_t *>(problem)->getEnvironment();
// get metric object
RCP<EvaluatePartition<basic_id_t> >metricObject =
rcp(Zoltan2_TestingFramework::EvaluatePartitionFactory::
newEvaluatePartition(reinterpret_cast<partitioning_problem_t*>
(problem), adapter_name, ia,
&zoltan2_parameters));
std::string metric_types[] = {"Metrics", "Graph Metrics"};
for (auto metric_type : metric_types) {
if( problem_parameters.isParameter(metric_type)) {
// calculate pass fail based on imbalance
if(rank == 0) cout << "Analyzing " << metric_type << endl;
if(rank == 0) {
std::cout << metric_type << " for " << problem_kind << ":" << std::endl;
if (metric_type == "Metrics")
metricObject->printMetrics(cout);
else if (metric_type == "Graph Metrics")
metricObject->printGraphMetrics(cout);
}
std::ostringstream msg;
auto metricsPlist = problem_parameters.sublist(metric_type); // get the metrics plist
bool all_tests_pass = false;
all_tests_pass
= MetricAnalyzer::analyzeMetrics( metricsPlist,
metricObject,
comm,
msg);
std::cout << msg.str() << std::endl;
if(rank == 0 && all_tests_pass) cout << "All " << metric_type << " tests PASSED." << endl;
else if (rank == 0) cout << "One or more " << metric_type << " tests FAILED." << endl;
} else if (rank == 0) cout << metric_type << " analysis unrequested. PASSED." << endl;
}
#define BDD
#ifdef BDD
if (problem_kind == "ordering") {
std::cout << "\nLet's examine the solution..." << std::endl;
auto solution = reinterpret_cast<ordering_problem_t *>(problem)->getSolution();
if (solution->haveSeparators() ) {
std::ostringstream sol;
sol << "Number of column blocks: " << solution->getNumSeparatorBlocks() << std::endl;
if (solution->getPermutationSize() < 100) {
if (solution->havePerm()) {
sol << "permutation: {";
for (auto &x : solution->getPermutationRCPConst(false)) sol << " " << x;
sol << "}" << std::endl;
}
if (solution->haveInverse()) {
sol << "inverse permutation: {";
for (auto &x : solution->getPermutationRCPConst(true)) sol << " " << x;
sol << "}" << std::endl;
}
if (solution->haveSeparatorRange()) {
sol << "separator range: {";
for (auto &x : solution->getSeparatorRangeRCPConst()) sol << " " << x;
sol << "}" << std::endl;
}
if (solution->haveSeparatorTree()) {
sol << "separator tree: {";
for (auto &x : solution->getSeparatorTreeRCPConst()) sol << " " << x;
sol << "}" << std::endl;
}
}
std::cout << sol.str() << std::endl;
}
}
#endif
// 4b. timers
// if(zoltan2_parameters.isParameter("timer_output_stream"))
// reinterpret_cast<partitioning_problem_t *>(problem)->printTimers();
////////////////////////////////////////////////////////////
// 5. Add solution to map for possible comparison testing
////////////////////////////////////////////////////////////
comparison_source->adapter = RCP<basic_id_t>(reinterpret_cast<basic_id_t *>(ia));
comparison_source->problem = RCP<base_problem_t>(reinterpret_cast<base_problem_t *>(problem));
comparison_source->metricObject = metricObject;
comparison_source->problem_kind = problem_parameters.isParameter("kind") ? problem_parameters.get<string>("kind") : "?";
comparison_source->adapter_kind = adapter_name;
// write mesh solution
// auto sol = reinterpret_cast<partitioning_problem_t *>(problem)->getSolution();
// MyUtils::writePartionSolution(sol.getPartListView(), ia->getLocalNumIDs(), comm);
////////////////////////////////////////////////////////////
// 6. Clean up
////////////////////////////////////////////////////////////
}
Teuchos::ParameterList * ML_Epetra::GetValidMLPParameters(){
using Teuchos::AnyNumberParameterEntryValidator;
using Teuchos::Array;
using Teuchos::ParameterList;
using Teuchos::setIntParameter;
using Teuchos::setDoubleParameter;
using Teuchos::setStringToIntegralParameter;
using Teuchos::tuple;
using std::string;
ParameterList dummy;
ParameterList * PL = new ParameterList;
// prevent Teuchos from converting parameter types
AnyNumberParameterEntryValidator::AcceptedTypes intParam(false),
dblParam(false),
strParam(false);
intParam.allowInt(true);
dblParam.allowDouble(true);
strParam.allowString(true);
/* Allocate List for Smoothing Options */
# if defined(HAVE_PETSC) && defined(HAVE_ML_SUPERLU4_0)
const int num_smoothers=30;
# elif defined(HAVE_PETSC) || defined(HAVE_ML_SUPERLU4_0)
const int num_smoothers=29;
#elif defined(HAVE_ML_TekoSmoothers)
const int num_smoothers=29; // won't work with SUPERLU or PETSC!
# else
const int num_smoothers=28;
# endif
const char* smoother_strings[num_smoothers]={"Aztec","IFPACK","Jacobi",
"ML symmetric Gauss-Seidel","symmetric Gauss-Seidel","ML Gauss-Seidel",
"Gauss-Seidel","block Gauss-Seidel","symmetric block Gauss-Seidel",
"Chebyshev","MLS","Hiptmair","Amesos-KLU","Amesos-Superlu",
"Amesos-UMFPACK","Amesos-Superludist","Amesos-MUMPS","user-defined",
"SuperLU","IFPACK-Chebyshev","self","do-nothing","IC","ICT","ILU","ILUT",
"Block Chebyshev","IFPACK-Block Chebyshev"
# ifdef HAVE_PETSC
,"petsc"
# endif
# ifdef HAVE_ML_SUPERLU4_0
,"SILU"
//#else
//#error "No SuperLU for you!"
# endif
# ifdef HAVE_ML_TekoSmoothers
,"teko"
# endif
};
Array<string> smoothers(num_smoothers);
for(int i = 0; i<num_smoothers; i++) {
smoothers[i] = smoother_strings[i];
}
/* General Options (Section 6.4.1) */
setIntParameter("ML output",0,"Output Level",PL,intParam);
setIntParameter("print unused",-2,"Print unused parameters",PL,intParam);
setIntParameter("ML print initial list",-2,"Print initial list supplied to constructor",PL,intParam);
setIntParameter("ML print final list",-2,"Print final list used by constructor",PL,intParam);
setIntParameter("PDE equations",1,"# of PDE equations per node",PL,intParam);
setStringToIntegralParameter<int>("eigen-analysis: type","cg","Scheme to compute spectral radius",
tuple<string>("cg","Anorm","power-method"),PL);
setIntParameter("eigen-analysis: iterations",10,"# iterations of eigen-anaysis",PL,intParam);
PL->set("ML label","dummy string");
setIntParameter("print hierarchy",-2,"Print hierarchy. 0 or greater prints individual levels.",PL,intParam);
/* Multigrid Cycle Options (Section 6.4.2) */
setIntParameter("cycle applications",1,"# MG cycles",PL,intParam);
setIntParameter("max levels",10,"Max # of levels",PL,intParam);
setStringToIntegralParameter<int>("increasing or decreasing", "increasing", "Level numbering",tuple<string>("increasing","decreasing"),PL);
setStringToIntegralParameter<int>("prec type", "MGV","Multigrid cycle type",tuple<string>("MGV","MGW","full-MGV","one-level-postsmoothing","two-level-additive","two-level-hybrid","two-level-hybrid2","projected MGV"),PL);
PL->set("projected mode",(double**)0);
setIntParameter("number of projected modes",0,"# of modes to be projected out before and after the V-cycle",PL,intParam);
/* Aggregation and Prolongator Options (Section 6.4.3) */
SetValidAggrParams(PL);
for (int i = 0; i < 10; ++i) {
char param[32];
sprintf(param,"aggregation: list (level %d)",i);
SetValidAggrParams(&(PL->sublist(param)));
}
PL->set("energy minimization: enable",false);
setIntParameter("energy minimization: type",2,"Norm to use for energy minimization",PL,intParam);
setDoubleParameter("energy minimization: droptol",0.0,"Drop tolerance for energy minimization",PL,dblParam);
PL->set("energy minimization: cheap",false);
/* Smoothing Options (Section 6.4.4) */
SetValidSmooParams(PL,smoothers);
for (int i = 0; i < 10; ++i) {
char param[32];
sprintf(param,"smoother: list (level %d)",i);
SetValidSmooParams(&(PL->sublist(param)),smoothers);
}
SetValidSmooParams(&(PL->sublist("coarse: list")),smoothers);
/* Load-balancing Options (Section 6.4.6) */
setIntParameter("repartition: enable",0,"Enable repartitioning",PL,intParam);
setStringToIntegralParameter<int>("repartition: partitioner","Zoltan","Repartitioning method",tuple<string>("Zoltan","ParMETIS"),PL);
setDoubleParameter("repartition: max min ratio",1.3,"Specifies desired maximum imbalance ratio",PL,dblParam);
setIntParameter("repartition: min per proc",512,"Specifies minimum # rows / processor",PL,intParam);
setIntParameter("repartition: put on single proc",5000,"Specifies max global problem to be put on one processor",PL,intParam);
setDoubleParameter("repartition: node max min ratio",1.3,"Specifies desired maximum imbalance for nodal heirarchy (Maxwell)",PL,dblParam);
setIntParameter("repartition: node min per proc",170,"Specifies minimum number of nodes per proc (Maxwell)",PL,intParam);
setIntParameter("repartition: Zoltan dimensions",0,"Dimension of problem",PL,intParam);
setIntParameter("repartition: start level",1,"Suppress repartitioning until this level",PL,intParam);
/* Analysis Options (Section 6.4.7) */
PL->set("analyze memory",false);
PL->set("viz: enable",false);
setStringToIntegralParameter<int>("viz: output format","vtk","Visualization format",tuple<string>("vtk","xyz","openx"),PL);
PL->set("viz: print starting solution",false);
setIntParameter("viz: equation to plot",-1,"Equation number to print",PL,intParam);
/* Miscellaneous Options (Section 6.4.8) */
PL->set("x-coordinates",(double*)0);
PL->set("y-coordinates",(double*)0);
PL->set("z-coordinates",(double*)0);
PL->set("node: x-coordinates",(double*)0);
PL->set("node: y-coordinates",(double*)0);
PL->set("node: z-coordinates",(double*)0);
PL->set("read XML",true);
PL->set("XML input file","ml_ParameterList.xml",string(""));
/* Smoothed Aggregation and the Null Space (Section 6.4.9) */
setStringToIntegralParameter<int>("null space: type","default vectors","Type of null space to use",tuple<string>("pre-computed","enriched","default vectors","elasticity from coordinates"),PL);
PL->set("null space: vectors",(double*)0);
setIntParameter("null space: dimension",0,"Number of user-supplied null space vectors",PL,intParam);
setIntParameter("null space: vectors to compute",1,"Number of vectors to compute",PL,intParam);
PL->set("null space: add default vectors",true);
/* Aggregation Strategies (Section 6.4.10) */
PL->set("aggregation: aux: enable",false);
setDoubleParameter("aggregation: aux: threshold",0.0,"Dropping threshold for auxillary matrix",PL,dblParam);
/* Unlisted Options */
PL->set("ML debug mode",false);
setStringToIntegralParameter<int>("default values","SA","Internal Option",tuple<string>("SA","DD","DD-ML","maxwell","NSSA","RefMaxwell","DD-ML-LU"),PL);
PL->set("ML validate parameter list",true);
setIntParameter("ML validate depth",0,"Internal option to control validation depth",PL,intParam);
PL->set("ResetList",true);
setStringToIntegralParameter<int>("SetDefaults","not-set","Internal Option",tuple<string>("not-set","SA","DD","DD-ML","maxwell","NSSA","RefMaxwell"),PL);
setIntParameter("ML node id",-1,"Experimental option to identify the processor node (vis-a-vis core) id",PL,intParam);
/* Unlisted Options that should probably be listed */
setIntParameter("aggregation: aux: max levels",10,"Unlisted option",PL,intParam);
PL->set("low memory usage",false);
setDoubleParameter("aggregation: edge prolongator drop threshold",0.0,"Unlisted option",PL,dblParam);
PL->set("zero starting solution",true);
PL->set("aggregation: block scaling",false);
setIntParameter("profile: operator iterations",0,"Unlisted option",PL,intParam);
setDoubleParameter("subsmoother: edge alpha",20.0,"alpha for edge Chebyshev polynomial in Hiptmair",PL,dblParam);
setDoubleParameter("subsmoother: node alpha",20.0,"alpha for node Chebyshev polynomial in Hiptmair",PL,dblParam);
PL->set("reuse: enable",false);
/* Unlisted options that should probably go away */
setIntParameter("output",0,"Output Level",PL,intParam);
/* Hightly experimental */
PL->set("repartition: output timings",false);
setIntParameter("repartition: estimated iterations",0,"Estimated number of iterations",PL,intParam);
setStringToIntegralParameter<int>("repartition: Zoltan type","RCB","Type of repartitioner to use",tuple<string>("RCB","hypergraph","fast hypergraph"),PL);
/* EXPERIMENTAL - Half-GS Smoothing */
PL->set("smoother: Gauss-Seidel efficient symmetric",false);
/* Coarse IFPACK Solvers - experimental */
PL->set("coarse: ifpack list",dummy);
PL->set("coarse: ifpack type",std::string(""));
setIntParameter("coarse: ifpack overlap",0,"Unlisted option",PL,intParam);
setDoubleParameter("coarse: ifpack level-of-fill",0.0,"Unlisted option",PL,dblParam);
setDoubleParameter("coarse: ifpack relative threshold",1.0,"Unlisted option",PL,dblParam);
setDoubleParameter("coarse: ifpack absolute threshold",0.0,"Unlisted option",PL,dblParam);
/* EXPERIMENTAL - RefMaxwell block parallelization */
PL->set("partitioner: options",dummy);
PL->sublist("partitioner: options").disableRecursiveValidation();
/* EXPERIMENTAL - node aware code */
setIntParameter("ML node id", -1, "Unlisted option", PL, intParam);
return PL;
}
int main(int argc, char *argv[])
{
typedef Teuchos::ScalarTraits<double> ST;
try
{
GlobalMPISession session(&argc, &argv);
MPIComm::world().synchronize();
VectorType<double> type = new EpetraVectorType();
#ifdef HAVE_CONFIG_H
ParameterXMLFileReader reader(Sundance::searchForFile("SolverParameters/userPrecParams.xml"));
#else
ParameterXMLFileReader reader("userPrecParams.xml");
#endif
ParameterList solverParams = reader.getParameters();
ParameterList innerSolverParams = solverParams.sublist("Inner Solve");
ParameterList outerSolverParams = solverParams.sublist("Outer Solve");
/* create the range space */
int nLocalRows = solverParams.get<int>("nLocal");
MatrixLaplacian1D builder(nLocalRows, type);
LinearOperator<double> A = builder.getOp();
Vector<double> x = A.domain().createMember();
int myRank = MPIComm::world().getRank();
int nProcs = MPIComm::world().getNProc();
Thyra::randomize(-ST::one(),+ST::one(),x.ptr().ptr());
#ifdef TRILINOS_6
if (myRank==0) x[0] = 0.0;
if (myRank==nProcs-1) x[nProcs * nLocalRows - 1] = 0.0;
// need to fix operator[] routine
#else
if (myRank==0) x.setElement(0, 0);
if (myRank==nProcs-1) x.setElement(nProcs * nLocalRows - 1, 0.0);
#endif
cout << "x=" << std::endl;
x.print(cout);
Vector<double> y = A*x;
cout << "y=" << std::endl;
y.print(cout);
Vector<double> ans = A.range().createMember();
LinearSolver<double> innerSolver
= LinearSolverBuilder::createSolver(innerSolverParams);
LinearSolver<double> outerSolver
= LinearSolverBuilder::createSolver(outerSolverParams);
/* call the setUserPrec() function to set the operator and solver
* to be used for preconditioning */
outerSolver.setUserPrec(A, innerSolver);
LinearOperator<double> AInv = inverse(A, outerSolver);
ans = AInv * y;
// SolverState<double> state = solver.solve(A, y, ans);
// cout << state << std::endl;
cout << "answer is " << std::endl;
ans.print(cout);
double err = (x-ans).norm2();
cout << "error norm = " << err << std::endl;
double tol = 1.0e-8;
if (err > tol)
{
cout << "User-defined preconditioner test FAILED" << std::endl;
return 1;
}
else
{
cout << "User-defined preconditioner test PASSED" << std::endl;
return 0;
}
}
catch(std::exception& e)
{
cout << "Caught exception: " << e.what() << std::endl;
return -1;
}
}
int
main (int argc, char *argv[])
{
using namespace TrilinosCouplings; // Yes, this means I'm lazy.
using TpetraIntrepidPoissonExample::exactResidualNorm;
using TpetraIntrepidPoissonExample::makeMatrixAndRightHandSide;
using TpetraIntrepidPoissonExample::solveWithBelos;
using TpetraIntrepidPoissonExample::solveWithBelosGPU;
using IntrepidPoissonExample::makeMeshInput;
using IntrepidPoissonExample::parseCommandLineArguments;
using IntrepidPoissonExample::setCommandLineArgumentDefaults;
using IntrepidPoissonExample::setMaterialTensorOffDiagonalValue;
using IntrepidPoissonExample::setUpCommandLineArguments;
using Tpetra::DefaultPlatform;
using Teuchos::Comm;
using Teuchos::outArg;
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcpFromRef;
using Teuchos::getFancyOStream;
using Teuchos::FancyOStream;
using std::endl;
// Pull in typedefs from the example's namespace.
typedef TpetraIntrepidPoissonExample::ST ST;
#ifdef HAVE_TRILINOSCOUPLINGS_MUELU
typedef TpetraIntrepidPoissonExample::LO LO;
typedef TpetraIntrepidPoissonExample::GO GO;
#endif // HAVE_TRILINOSCOUPLINGS_MUELU
typedef TpetraIntrepidPoissonExample::Node Node;
typedef Teuchos::ScalarTraits<ST> STS;
typedef STS::magnitudeType MT;
typedef Teuchos::ScalarTraits<MT> STM;
typedef TpetraIntrepidPoissonExample::sparse_matrix_type sparse_matrix_type;
typedef TpetraIntrepidPoissonExample::vector_type vector_type;
typedef TpetraIntrepidPoissonExample::operator_type operator_type;
bool success = true;
try {
Teuchos::oblackholestream blackHole;
Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
const int myRank = mpiSession.getRank ();
//const int numProcs = mpiSession.getNProc ();
// Get the default communicator and Kokkos Node instance
RCP<const Comm<int> > comm =
DefaultPlatform::getDefaultPlatform ().getComm ();
RCP<Node> node = DefaultPlatform::getDefaultPlatform ().getNode ();
// Did the user specify --help at the command line to print help
// with command-line arguments?
bool printedHelp = false;
// Values of command-line arguments.
int nx, ny, nz;
std::string xmlInputParamsFile;
bool verbose, debug;
int maxNumItersFromCmdLine = -1; // -1 means "read from XML file"
double tolFromCmdLine = -1.0; // -1 means "read from XML file"
std::string solverName = "GMRES";
ST materialTensorOffDiagonalValue = 0.0;
// Set default values of command-line arguments.
setCommandLineArgumentDefaults (nx, ny, nz, xmlInputParamsFile,
solverName, verbose, debug);
// Parse and validate command-line arguments.
Teuchos::CommandLineProcessor cmdp (false, true);
setUpCommandLineArguments (cmdp, nx, ny, nz, xmlInputParamsFile,
solverName, tolFromCmdLine,
maxNumItersFromCmdLine,
verbose, debug);
cmdp.setOption ("materialTensorOffDiagonalValue",
&materialTensorOffDiagonalValue, "Off-diagonal value in "
"the material tensor. This controls the iteration count. "
"Be careful with this if you use CG, since you can easily "
"make the matrix indefinite.");
// Additional command-line arguments for GPU experimentation.
bool gpu = false;
cmdp.setOption ("gpu", "no-gpu", &gpu,
"Run example using GPU node (if supported)");
int ranks_per_node = 1;
cmdp.setOption ("ranks_per_node", &ranks_per_node,
"Number of MPI ranks per node");
int gpu_ranks_per_node = 1;
cmdp.setOption ("gpu_ranks_per_node", &gpu_ranks_per_node,
"Number of MPI ranks per node for GPUs");
int device_offset = 0;
cmdp.setOption ("device_offset", &device_offset,
"Offset for attaching MPI ranks to CUDA devices");
// Additional command-line arguments for dumping the generated
// matrix or its row Map to output files.
//
// FIXME (mfh 09 Apr 2014) Need to port these command-line
// arguments to the Epetra version.
// If matrixFilename is nonempty, dump the matrix to that file
// in MatrixMarket format.
std::string matrixFilename;
cmdp.setOption ("matrixFilename", &matrixFilename, "If nonempty, dump the "
"generated matrix to that file in MatrixMarket format.");
// If rowMapFilename is nonempty, dump the matrix's row Map to
// that file in MatrixMarket format.
std::string rowMapFilename;
cmdp.setOption ("rowMapFilename", &rowMapFilename, "If nonempty, dump the "
"generated matrix's row Map to that file in a format that "
"Tpetra::MatrixMarket::Reader can read.");
// Option to exit after building A and b (and dumping stuff to
// files, if requested).
bool exitAfterAssembly = false;
cmdp.setOption ("exitAfterAssembly", "dontExitAfterAssembly",
&exitAfterAssembly, "If true, exit after building the "
"sparse matrix and dense right-hand side vector. If either"
" --matrixFilename or --rowMapFilename are nonempty strings"
", dump the matrix resp. row Map to their respective files "
"before exiting.");
parseCommandLineArguments (cmdp, printedHelp, argc, argv, nx, ny, nz,
xmlInputParamsFile, solverName, verbose, debug);
if (printedHelp) {
// The user specified --help at the command line to print help
// with command-line arguments. We printed help already, so quit
// with a happy return code.
return EXIT_SUCCESS;
}
setMaterialTensorOffDiagonalValue (materialTensorOffDiagonalValue);
// Both streams only print on MPI Rank 0. "out" only prints if the
// user specified --verbose.
RCP<FancyOStream> out =
getFancyOStream (rcpFromRef ((myRank == 0 && verbose) ? std::cout : blackHole));
RCP<FancyOStream> err =
getFancyOStream (rcpFromRef ((myRank == 0 && debug) ? std::cerr : blackHole));
#ifdef HAVE_MPI
*out << "PARALLEL executable" << endl;
#else
*out << "SERIAL executable" << endl;
#endif
/**********************************************************************************/
/********************************** GET XML INPUTS ********************************/
/**********************************************************************************/
ParameterList inputList;
if (xmlInputParamsFile != "") {
*out << "Reading parameters from XML file \""
<< xmlInputParamsFile << "\"..." << endl;
Teuchos::updateParametersFromXmlFile (xmlInputParamsFile,
outArg (inputList));
if (myRank == 0) {
inputList.print (*out, 2, true, true);
*out << endl;
}
}
// Get Pamgen mesh definition string, either from the input
// ParameterList or from our function that makes a cube and fills in
// the number of cells along each dimension.
std::string meshInput = inputList.get("meshInput", "");
if (meshInput == "") {
*out << "Generating mesh input string: nx = " << nx
<< ", ny = " << ny
<< ", nz = " << nz << endl;
meshInput = makeMeshInput (nx, ny, nz);
}
// Total application run time
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Time", total_time);
RCP<sparse_matrix_type> A;
RCP<vector_type> B, X_exact, X;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Assembly", total_assembly);
makeMatrixAndRightHandSide (A, B, X_exact, X, comm, node, meshInput,
out, err, verbose, debug);
}
// Optionally dump the matrix and/or its row Map to files.
{
typedef Tpetra::MatrixMarket::Writer<sparse_matrix_type> writer_type;
if (matrixFilename != "") {
writer_type::writeSparseFile (matrixFilename, A);
}
if (rowMapFilename != "") {
writer_type::writeMapFile (rowMapFilename, * (A->getRowMap ()));
}
}
if (exitAfterAssembly) {
// Users might still be interested in assembly time.
Teuchos::TimeMonitor::report (comm.ptr (), std::cout);
return EXIT_SUCCESS;
}
const std::vector<MT> norms = exactResidualNorm (A, B, X_exact);
// X_exact is the exact solution of the PDE, projected onto the
// discrete mesh. It may not necessarily equal the exact solution
// of the linear system.
*out << "||B - A*X_exact||_2 = " << norms[0] << endl
<< "||B||_2 = " << norms[1] << endl
<< "||A||_F = " << norms[2] << endl;
// Setup preconditioner
std::string prec_type = inputList.get ("Preconditioner", "None");
RCP<operator_type> M;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Preconditioner Setup", total_prec);
if (prec_type == "MueLu") {
#ifdef HAVE_TRILINOSCOUPLINGS_MUELU
if (inputList.isSublist("MueLu")) {
ParameterList mueluParams = inputList.sublist("MueLu");
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A,mueluParams);
} else {
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A);
}
#else // NOT HAVE_TRILINOSCOUPLINGS_MUELU
TEUCHOS_TEST_FOR_EXCEPTION(
prec_type == "MueLu", std::runtime_error, "Tpetra scaling example: "
"In order to precondition with MueLu, you must have built Trilinos "
"with the MueLu package enabled.");
#endif // HAVE_TRILINOSCOUPLINGS_MUELU
}
} // setup preconditioner
// Get the convergence tolerance for each linear solve.
// If the user provided a nonnegative value at the command
// line, it overrides any value in the input ParameterList.
MT tol = STM::squareroot (STM::eps ()); // default value
if (tolFromCmdLine < STM::zero ()) {
tol = inputList.get ("Convergence Tolerance", tol);
} else {
tol = tolFromCmdLine;
}
// Get the maximum number of iterations for each linear solve.
// If the user provided a value other than -1 at the command
// line, it overrides any value in the input ParameterList.
int maxNumIters = 200; // default value
if (maxNumItersFromCmdLine == -1) {
maxNumIters = inputList.get ("Maximum Iterations", maxNumIters);
} else {
maxNumIters = maxNumItersFromCmdLine;
}
// Get the number of "time steps." We imitate a time-dependent
// PDE by doing this many linear solves.
const int num_steps = inputList.get ("Number of Time Steps", 1);
// Do the linear solve(s).
bool converged = false;
int numItersPerformed = 0;
if (gpu) {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total GPU Solve", total_solve);
solveWithBelosGPU (converged, numItersPerformed, tol, maxNumIters,
num_steps, ranks_per_node, gpu_ranks_per_node,
device_offset, prec_type, X, A, B, Teuchos::null, M);
}
else {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Solve", total_solve);
solveWithBelos (converged, numItersPerformed, solverName, tol,
maxNumIters, num_steps, X, A, B, Teuchos::null, M);
}
// Compute ||X-X_exact||_2
const MT norm_x = X_exact->norm2 ();
X_exact->update (-1.0, *X, 1.0);
const MT norm_error = X_exact->norm2 ();
*out << endl
<< "||X - X_exact||_2 / ||X_exact||_2 = " << norm_error / norm_x
<< endl;
} // total time block
// Summarize timings
Teuchos::TimeMonitor::report (comm.ptr (), std::cout);
} // try
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
if (success) {
return EXIT_SUCCESS;
} else {
return EXIT_FAILURE;
}
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::TimeMonitor;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
bool success = false;
bool verbose = true;
try {
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int numProc = comm->getSize();
int myRank = comm->getRank();
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& out = *fancy;
out.setOutputToRootOnly(0);
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
Xpetra::Parameters xpetraParameters(clp);
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
const int numLists = 1;
std::vector<std::string> dirList;
dirList.push_back("Convergence/Laplace2D/");
bool failed = false;
for (int k = 0; k < numLists; k++) {
const std::string& dirName = dirList[k];
std::string problemFile = dirName + "problem.xml";
ParameterList galeriParameters;
Teuchos::updateParametersFromXmlFileAndBroadcast(problemFile, Teuchos::Ptr<Teuchos::ParameterList>(&galeriParameters), *comm);
if (!galeriParameters.isParameter("mz"))
galeriParameters.set<int>("mz", -1);
// =========================================================================
// Problem construction (copy-paste from Driver.cpp)
// =========================================================================
RCP<Matrix> A;
RCP<Map> map;
RCP<MultiVector> nullspace, coordinates;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriParameters.
std::string matrixType = galeriParameters.get<std::string>("matrixType");
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian1D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriParameters);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian2D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriParameters);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(lib, "Cartesian3D", comm, galeriParameters);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriParameters);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
#if 0
out << "========================================================\n" << xpetraParameters << galeriParameters;
out << "Processor subdomains in x direction: " << galeriParameters.get<GO>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriParameters.get<GO>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriParameters.get<GO>("mz") << std::endl
<< "========================================================" << std::endl;
#endif
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixType, map, galeriParameters);
A = Pr->BuildMatrix();
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((matrixType == "Elasticity2D") ? 2 : 3);
} else {
nullspace = MultiVectorFactory::Build(map, 1);
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(0);
for (int i = 0; i < nsData.size(); i++)
nsData[i] = one;
}
// =========================================================================
// Run different configurations
// =========================================================================
Teuchos::ArrayRCP<std::string> fileList = MueLuTests::TestHelpers::GetFileList(dirList[k],
(numProc == 1 ? std::string(".xml") : std::string("_np" + Teuchos::toString(numProc) + ".xml")));
RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
for (int i = 0; i < fileList.size(); i++) {
if (fileList[i] == "problem.xml")
continue;
// Set seed
Utilities::SetRandomSeed(*comm);
// Reset (potentially) cached value of the estimate
A->SetMaxEigenvalueEstimate(-Teuchos::ScalarTraits<SC>::one());
std::string xmlFile = dirName + fileList[i];
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFile, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
std::string solveType = paramList.get<std::string> ("solver", "standalone");
double goldRate = paramList.get<double> ("convergence rate");
ParameterList& mueluList = paramList.sublist ("MueLu");
TEUCHOS_TEST_FOR_EXCEPTION(solveType != "standalone" && solveType != "cg" && solveType != "gmres", MueLu::Exceptions::RuntimeError,
"Unknown solver type \"" << solveType << "\"");
bool isPrec = !(solveType == "standalone");
if (!mueluList.isParameter("verbosity"))
mueluList.set("verbosity", "none");
#ifndef HAVE_MUELU_BELOS
if (isPrec)
out << xmlFile << ": skipped (Belos is not enabled)" << std::endl;
#endif
// =========================================================================
// Preconditioner construction
// =========================================================================
RCP<Hierarchy> H;
try {
ParameterListInterpreter mueluFactory(mueluList);
H = mueluFactory.CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
H->GetLevel(0)->Set("Coordinates", coordinates);
mueluFactory.SetupHierarchy(*H);
} catch (Teuchos::ExceptionBase& e) {
std::string msg = e.what();
msg = msg.substr(msg.find_last_of('\n')+1);
out << "Caught exception: " << msg << std::endl;
if (msg == "Zoltan interface is not available" ||
msg == "Zoltan2 interface is not available") {
if (myRank == 0)
out << xmlFile << ": skipped (missing library)" << std::endl;
continue;
}
}
// Set X, B
{
// TODO: do multiple vectors simultaneously to average
// we set seed for reproducibility
Utilities::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
const int maxIts = 100;
const double tol = 1e-12;
H->IsPreconditioner(isPrec);
if (isPrec == false) {
MueLu::ReturnType ret = H->Iterate(*B, *X, std::pair<LO,SC>(maxIts, tol));
double rate = H->GetRate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == MueLu::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == MueLu::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} else {
#ifdef HAVE_MUELU_BELOS
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
// Define Operator and Preconditioner
RCP<OP> belosOp = rcp(new Belos::XpetraOp<SC, LO, GO, NO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
RCP<OP> belosPrec = rcp(new Belos::MueLuOp <SC, LO, GO, NO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
out << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
#if 1
belosList.set("Verbosity", Belos::Errors + Belos::Warnings);
#else
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
#endif
// Belos custom test to store residuals
// Create an iterative solver manager
RCP<Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg")
solver = rcp(new Belos::PseudoBlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
else if (solveType == "gmres")
solver = rcp(new Belos::BlockGmresSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
RCP<Belos::MyStatusTest<SC, MV, OP> > status = rcp(new Belos::MyStatusTest<SC, MV, OP>(tol));
solver->setDebugStatusTest(status);
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
double rate = status->rate();
if (abs(rate-goldRate) < 0.02) {
out << xmlFile << ": passed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
} else {
out << xmlFile << ": failed (" <<
(ret == Belos::Converged ? "converged, " : "unconverged, ") <<
"expected rate = " << goldRate << ", real rate = " << rate <<
(ret == Belos::Converged ? "" : " (after " + Teuchos::toString(maxIts) + " iterations)")
<< ")" << std::endl;
failed = true;
}
} catch(...) {
out << xmlFile << ": failed (exception)" << std::endl;
failed = true;
}
#endif //ifdef HAVE_MUELU_BELOS
}
}
}
success = !failed;
out << std::endl << "End Result: TEST " << (failed ? "FAILED" : "PASSED") << std::endl;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
return ( success ? EXIT_SUCCESS : EXIT_FAILURE );
}
TEUCHOS_UNIT_TEST(PdQuickGridDiscretization_MPI_np2, SimpleTensorProductMeshTest) {
Teuchos::RCP<Epetra_Comm> comm;
comm = rcp(new Epetra_MpiComm(MPI_COMM_WORLD));
int numProcs = comm->NumProc();
int rank = comm->MyPID();
TEST_COMPARE(numProcs, ==, 2);
if(numProcs != 2){
std::cerr << "Unit test runtime ERROR: utPeridigm_PdQuickGridDiscretization_MPI_np2 only makes sense on 2 processors" << std::endl;
return;
}
RCP<ParameterList> discParams = rcp(new ParameterList);
// create a 2x2x2 discretization
// specify a spherical neighbor search with the horizon a tad longer than the mesh spacing
discParams->set("Type", "PdQuickGrid");
discParams->set("NeighborhoodType", "Spherical");
ParameterList& quickGridParams = discParams->sublist("TensorProduct3DMeshGenerator");
quickGridParams.set("Type", "PdQuickGrid");
quickGridParams.set("X Origin", 0.0);
quickGridParams.set("Y Origin", 0.0);
quickGridParams.set("Z Origin", 0.0);
quickGridParams.set("X Length", 1.0);
quickGridParams.set("Y Length", 1.0);
quickGridParams.set("Z Length", 1.0);
quickGridParams.set("Number Points X", 2);
quickGridParams.set("Number Points Y", 2);
quickGridParams.set("Number Points Z", 2);
// initialize the horizon manager and set the horizon to 0.501
ParameterList blockParameterList;
ParameterList& blockParams = blockParameterList.sublist("My Block");
blockParams.set("Block Names", "block_1");
blockParams.set("Horizon", 0.501);
PeridigmNS::HorizonManager::self().loadHorizonInformationFromBlockParameters(blockParameterList);
// create the discretization
RCP<PdQuickGridDiscretization> discretization =
rcp(new PdQuickGridDiscretization(comm, discParams));
// sanity check, calling with a dimension other than 1 or 3 should throw an exception
TEST_THROW(discretization->getGlobalOwnedMap(0), Teuchos::Exceptions::InvalidParameter);
TEST_THROW(discretization->getGlobalOwnedMap(2), Teuchos::Exceptions::InvalidParameter);
TEST_THROW(discretization->getGlobalOwnedMap(4), Teuchos::Exceptions::InvalidParameter);
// basic checks on the 1d map
Teuchos::RCP<const Epetra_BlockMap> map = discretization->getGlobalOwnedMap(1);
TEST_ASSERT(map->NumGlobalElements() == 8);
TEST_ASSERT(map->NumMyElements() == 4);
TEST_ASSERT(map->ElementSize() == 1);
TEST_ASSERT(map->IndexBase() == 0);
TEST_ASSERT(map->UniqueGIDs() == true);
int* myGlobalElements = map->MyGlobalElements();
if(rank == 0){
TEST_ASSERT(myGlobalElements[0] == 0);
TEST_ASSERT(myGlobalElements[1] == 2);
TEST_ASSERT(myGlobalElements[2] == 4);
TEST_ASSERT(myGlobalElements[3] == 6);
}
if(rank == 1){
TEST_ASSERT(myGlobalElements[0] == 5);
TEST_ASSERT(myGlobalElements[1] == 7);
TEST_ASSERT(myGlobalElements[2] == 1);
TEST_ASSERT(myGlobalElements[3] == 3);
}
// check the 1d overlap map
// for this simple discretization, everything should be ghosted on both processors
Teuchos::RCP<const Epetra_BlockMap> overlapMap = discretization->getGlobalOverlapMap(1);
TEST_ASSERT(overlapMap->NumGlobalElements() == 16);
TEST_ASSERT(overlapMap->NumMyElements() == 8);
TEST_ASSERT(overlapMap->ElementSize() == 1);
TEST_ASSERT(overlapMap->IndexBase() == 0);
TEST_ASSERT(overlapMap->UniqueGIDs() == false);
myGlobalElements = overlapMap->MyGlobalElements();
if(rank == 0){
TEST_ASSERT(myGlobalElements[0] == 0);
TEST_ASSERT(myGlobalElements[1] == 2);
TEST_ASSERT(myGlobalElements[2] == 4);
TEST_ASSERT(myGlobalElements[3] == 6);
TEST_ASSERT(myGlobalElements[4] == 1);
TEST_ASSERT(myGlobalElements[5] == 3);
TEST_ASSERT(myGlobalElements[6] == 5);
TEST_ASSERT(myGlobalElements[7] == 7);
}
if(rank == 1){
TEST_ASSERT(myGlobalElements[0] == 5);
TEST_ASSERT(myGlobalElements[1] == 7);
TEST_ASSERT(myGlobalElements[2] == 1);
TEST_ASSERT(myGlobalElements[3] == 3);
TEST_ASSERT(myGlobalElements[4] == 0);
TEST_ASSERT(myGlobalElements[5] == 2);
TEST_ASSERT(myGlobalElements[6] == 4);
TEST_ASSERT(myGlobalElements[7] == 6);
}
// same checks for 3d map
map = discretization->getGlobalOwnedMap(3);
TEST_ASSERT(map->NumGlobalElements() == 8);
TEST_ASSERT(map->NumMyElements() == 4);
TEST_ASSERT(map->ElementSize() == 3);
TEST_ASSERT(map->IndexBase() == 0);
TEST_ASSERT(map->UniqueGIDs() == true);
myGlobalElements = map->MyGlobalElements();
if(rank == 0){
TEST_ASSERT(myGlobalElements[0] == 0);
TEST_ASSERT(myGlobalElements[1] == 2);
TEST_ASSERT(myGlobalElements[2] == 4);
TEST_ASSERT(myGlobalElements[3] == 6);
}
if(rank == 1){
TEST_ASSERT(myGlobalElements[0] == 5);
TEST_ASSERT(myGlobalElements[1] == 7);
TEST_ASSERT(myGlobalElements[2] == 1);
TEST_ASSERT(myGlobalElements[3] == 3);
}
// check the 3d overlap map
// for this simple discretization, everything should be ghosted on both processors
overlapMap = discretization->getGlobalOverlapMap(3);
TEST_ASSERT(overlapMap->NumGlobalElements() == 16);
TEST_ASSERT(overlapMap->NumMyElements() == 8);
TEST_ASSERT(overlapMap->ElementSize() == 3);
TEST_ASSERT(overlapMap->IndexBase() == 0);
TEST_ASSERT(overlapMap->UniqueGIDs() == false);
myGlobalElements = overlapMap->MyGlobalElements();
if(rank == 0){
TEST_ASSERT(myGlobalElements[0] == 0);
TEST_ASSERT(myGlobalElements[1] == 2);
TEST_ASSERT(myGlobalElements[2] == 4);
TEST_ASSERT(myGlobalElements[3] == 6);
TEST_ASSERT(myGlobalElements[4] == 1);
TEST_ASSERT(myGlobalElements[5] == 3);
TEST_ASSERT(myGlobalElements[6] == 5);
TEST_ASSERT(myGlobalElements[7] == 7);
}
if(rank == 1){
TEST_ASSERT(myGlobalElements[0] == 5);
TEST_ASSERT(myGlobalElements[1] == 7);
TEST_ASSERT(myGlobalElements[2] == 1);
TEST_ASSERT(myGlobalElements[3] == 3);
TEST_ASSERT(myGlobalElements[4] == 0);
TEST_ASSERT(myGlobalElements[5] == 2);
TEST_ASSERT(myGlobalElements[6] == 4);
TEST_ASSERT(myGlobalElements[7] == 6);
}
// check the bond map
// the horizon was chosen such that each point should have three neighbors
// note that if the NeighborhoodType parameter is not set to Spherical, this will fail
Teuchos::RCP<const Epetra_BlockMap> bondMap = discretization->getGlobalBondMap();
TEST_ASSERT(bondMap->NumGlobalElements() == 8);
TEST_ASSERT(bondMap->NumMyElements() == 4);
TEST_ASSERT(bondMap->IndexBase() == 0);
TEST_ASSERT(bondMap->UniqueGIDs() == true);
myGlobalElements = bondMap->MyGlobalElements();
if(rank == 0){
TEST_ASSERT(myGlobalElements[0] == 0);
TEST_ASSERT(myGlobalElements[1] == 2);
TEST_ASSERT(myGlobalElements[2] == 4);
TEST_ASSERT(myGlobalElements[3] == 6);
}
if(rank == 1){
TEST_ASSERT(myGlobalElements[0] == 5);
TEST_ASSERT(myGlobalElements[1] == 7);
TEST_ASSERT(myGlobalElements[2] == 1);
TEST_ASSERT(myGlobalElements[3] == 3);
}
TEST_ASSERT(discretization->getNumBonds() == 4*3);
// check the initial positions
// all three coordinates are contained in a single vector
Teuchos::RCP<Epetra_Vector> initialX = discretization->getInitialX();
TEST_ASSERT(initialX->MyLength() == 4*3);
TEST_ASSERT(initialX->GlobalLength() == 8*3);
if(rank == 0){
TEST_FLOATING_EQUALITY((*initialX)[0], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[1], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[2], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[3], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[4], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[5], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[6], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[7], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[8], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[9], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[10], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[11], 0.75, 1.0e-16);
}
if(rank == 1){
TEST_FLOATING_EQUALITY((*initialX)[0], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[1], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[2], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[3], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[4], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[5], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[6], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[7], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[8], 0.25, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[9], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[10], 0.75, 1.0e-16);
TEST_FLOATING_EQUALITY((*initialX)[11], 0.25, 1.0e-16);
}
// check cell volumes
Teuchos::RCP<Epetra_Vector> volume = discretization->getCellVolume();
TEST_ASSERT(volume->MyLength() == 4);
TEST_ASSERT(volume->GlobalLength() == 8);
for(int i=0 ; i<volume->MyLength() ; ++i)
TEST_FLOATING_EQUALITY((*volume)[i], 0.125, 1.0e-16);
// check the neighbor lists
Teuchos::RCP<PeridigmNS::NeighborhoodData> neighborhoodData = discretization->getNeighborhoodData();
TEST_ASSERT(neighborhoodData->NumOwnedPoints() == 4);
int* ownedIds = neighborhoodData->OwnedIDs();
TEST_ASSERT(ownedIds[0] == 0);
TEST_ASSERT(ownedIds[1] == 1);
TEST_ASSERT(ownedIds[2] == 2);
TEST_ASSERT(ownedIds[3] == 3);
TEST_ASSERT(neighborhoodData->NeighborhoodListSize() == 16);
int* neighborhood = neighborhoodData->NeighborhoodList();
int* neighborhoodPtr = neighborhoodData->NeighborhoodPtr();
// remember, these are local IDs on each processor,
// which includes both owned and ghost nodes (confusing!)
if(rank == 0){
TEST_ASSERT(neighborhoodPtr[0] == 0);
TEST_ASSERT(neighborhood[0] == 3);
TEST_ASSERT(neighborhood[1] == 4);
TEST_ASSERT(neighborhood[2] == 1);
TEST_ASSERT(neighborhood[3] == 2);
TEST_ASSERT(neighborhoodPtr[1] == 4);
TEST_ASSERT(neighborhood[4] == 3);
TEST_ASSERT(neighborhood[5] == 0);
TEST_ASSERT(neighborhood[6] == 5);
TEST_ASSERT(neighborhood[7] == 3);
TEST_ASSERT(neighborhoodPtr[2] == 8);
TEST_ASSERT(neighborhood[8] == 3);
TEST_ASSERT(neighborhood[9] == 0);
TEST_ASSERT(neighborhood[10] == 6);
TEST_ASSERT(neighborhood[11] == 3);
TEST_ASSERT(neighborhoodPtr[3] == 12);
TEST_ASSERT(neighborhood[12] == 3);
TEST_ASSERT(neighborhood[13] == 1);
TEST_ASSERT(neighborhood[14] == 2);
TEST_ASSERT(neighborhood[15] == 7);
}
if(rank == 1){
TEST_ASSERT(neighborhoodPtr[0] == 0);
TEST_ASSERT(neighborhood[0] == 3);
TEST_ASSERT(neighborhood[1] == 2);
TEST_ASSERT(neighborhood[2] == 6);
TEST_ASSERT(neighborhood[3] == 1);
TEST_ASSERT(neighborhoodPtr[1] == 4);
TEST_ASSERT(neighborhood[4] == 3);
TEST_ASSERT(neighborhood[5] == 3);
TEST_ASSERT(neighborhood[6] == 0);
TEST_ASSERT(neighborhood[7] == 7);
TEST_ASSERT(neighborhoodPtr[2] == 8);
TEST_ASSERT(neighborhood[8] == 3);
TEST_ASSERT(neighborhood[9] == 4);
TEST_ASSERT(neighborhood[10] == 3);
TEST_ASSERT(neighborhood[11] == 0);
TEST_ASSERT(neighborhoodPtr[3] == 12);
TEST_ASSERT(neighborhood[12] == 3);
TEST_ASSERT(neighborhood[13] == 2);
TEST_ASSERT(neighborhood[14] == 5);
TEST_ASSERT(neighborhood[15] == 1);
}
}
int main(int argc, char *argv[]) {
// Define default types
typedef double scalar_type;
typedef int local_ordinal_type;
typedef int global_ordinal_type;
typedef KokkosClassic::DefaultNode::DefaultNodeType node_type;
// Convenient typedef's
typedef Tpetra::Operator<scalar_type,local_ordinal_type,global_ordinal_type,node_type> operator_type;
typedef Tpetra::CrsMatrix<scalar_type,local_ordinal_type,global_ordinal_type,node_type> crs_matrix_type;
typedef Tpetra::RowMatrix<scalar_type,local_ordinal_type,global_ordinal_type,node_type> row_matrix_type;
typedef Tpetra::Vector<scalar_type,local_ordinal_type,global_ordinal_type,node_type> vector_type;
typedef Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_type,node_type> multivector_type;
typedef Tpetra::Map<local_ordinal_type,global_ordinal_type,node_type> driver_map_type;
typedef MueLu::TpetraOperator<scalar_type, local_ordinal_type, global_ordinal_type, node_type> muelu_tpetra_operator_type;
typedef MueLu::Utilities<scalar_type,local_ordinal_type,global_ordinal_type,node_type> MueLuUtilities;
typedef Belos::LinearProblem<scalar_type, multivector_type, operator_type> linear_problem_type;
typedef Belos::SolverManager<scalar_type, multivector_type, operator_type> belos_solver_manager_type;
typedef Belos::PseudoBlockCGSolMgr<scalar_type, multivector_type, operator_type> belos_pseudocg_manager_type;
typedef Belos::BlockGmresSolMgr<scalar_type, multivector_type, operator_type> belos_gmres_manager_type;
typedef Belos::BiCGStabSolMgr<scalar_type, multivector_type, operator_type> belos_bicgstab_manager_type;
typedef Ifpack2::Preconditioner<scalar_type, local_ordinal_type, global_ordinal_type, node_type> precond_type;
//MueLu_UseShortNames.hpp wants these typedefs.
typedef scalar_type Scalar;
typedef local_ordinal_type LocalOrdinal;
typedef global_ordinal_type GlobalOrdinal;
typedef node_type Node;
# include <MueLu_UseShortNames.hpp>
typedef Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> GaleriXpetraProblem;
typedef ADR::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> ADRXpetraProblem;
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp; // reference count pointers
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int mypid = comm->getRank();
/*
int subCommRank[3]={0,1,2};
Teuchos::ArrayView<int> arraySubCommRank(subCommRank, 3);
auto subComm = comm->createSubcommunicator(arraySubCommRank);
*/
Teuchos::CommandLineProcessor clp(false);
global_ordinal_type maxIts = 10000;
scalar_type tol = 1e-10;
std::string solverOptionsFile = "final_parser.xml";
std::string krylovSolverType = "bicgstab";
clp.setOption("xmlFile", &solverOptionsFile, "XML file containing MueLu solver parameters");
clp.setOption("maxits", &maxIts, "maximum number of Krylov iterations");
clp.setOption("tol", &tol, "tolerance for Krylov solver");
clp.setOption("krylovType", &krylovSolverType, "cg or gmres solver");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
ParameterList xmlParams;
ParameterList mueluParams;
ParameterList problemParams;
Teuchos::updateParametersFromXmlFile(solverOptionsFile, Teuchos::inoutArg(xmlParams));
mueluParams = xmlParams.sublist(static_cast<const std::string>("MueLu"));
problemParams = xmlParams.sublist(static_cast<const std::string>("Problem"));
// Problem definition
std::string problem_type = problemParams.get<std::string>(static_cast<const std::string>("problem type"));
// Parameters
Scalar Lx = problemParams.get<scalar_type>(static_cast<const std::string>("domain size in x-direction"));
Scalar Ly = problemParams.get<scalar_type>(static_cast<const std::string>("domain size in y-direction"));
Scalar Lz = problemParams.get<scalar_type>(static_cast<const std::string>("domain size in z-direction"));
global_ordinal_type nx = problemParams.get<int>(static_cast<const std::string>("nodes in x-direction"));
global_ordinal_type ny = problemParams.get<int>(static_cast<const std::string>("nodes in y-direction"));
global_ordinal_type nz = problemParams.get<int>(static_cast<const std::string>("nodes in z-direction"));
global_ordinal_type number_runs = problemParams.get<int>(static_cast<const std::string>("number of runs"));
MueLu::DomainPartitioning domain;
int keep_boundary = 0;
Scalar stretchx = (Scalar) Lx/nx;
Scalar stretchy = (Scalar) Ly/ny;
Scalar stretchz = (Scalar) Lz/nz;
ADR::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, problem_type, keep_boundary , stretchx, stretchy, stretchz); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
//
// Construct the problem
//
global_ordinal_type indexBase = 0;
RCP<const Map> xpetraMap;
std::vector<global_ordinal_type> ind;
//BRICK SIZE
int brick_sizex = mueluParams.get<int>(static_cast<const std::string>("aggregation: brick x size"));
int brick_sizey = mueluParams.get<int>(static_cast<const std::string>("aggregation: brick y size"));
int brick_sizez = mueluParams.get<int>(static_cast<const std::string>("aggregation: brick z size"));
//Creation of the map where processor 0 gets nothing at the fine level
if( comm->getSize()>1 )
{
if(problem_type == "ADR1D")
createBrickMap1D(matrixParameters.GetNumGlobalElements(), ind, comm );
else if(problem_type == "ADR2D")
createBrickMap2D( nx, brick_sizex, brick_sizey, ind, comm );
else if(problem_type == "ADR3D")
createBrickMap3D( nx, ny, brick_sizex, brick_sizey, brick_sizez, ind, comm );
ind.shrink_to_fit();
Teuchos::ArrayView<const global_ordinal_type> elementList (ind);
xpetraMap = MapFactory::Build(Xpetra::UseTpetra,matrixParameters.GetNumGlobalElements(), elementList, indexBase, comm);
}
else if( comm->getSize()==1 )
xpetraMap = MapFactory::Build(Xpetra::UseTpetra, matrixParameters.GetNumGlobalElements(), indexBase, comm);
RCP<MultiVector> coordinates;
if (problem_type == "ADR1D")
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<scalar_type,local_ordinal_type,global_ordinal_type,Map,MultiVector>("1D", xpetraMap, matrixParameters.GetParameterList());
else if (problem_type == "ADR2D")
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<scalar_type,local_ordinal_type,global_ordinal_type,Map,MultiVector>("2D", xpetraMap, matrixParameters.GetParameterList());
else if (problem_type == "ADR3D")
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<scalar_type,local_ordinal_type,global_ordinal_type,Map,MultiVector>("3D", xpetraMap, matrixParameters.GetParameterList());
RCP<ADRXpetraProblem> Pr = ADR::Xpetra::BuildProblem<scalar_type, local_ordinal_type, global_ordinal_type, Map, CrsMatrixWrap, MultiVector>(matrixParameters.GetMatrixType(), xpetraMap, matrixParameters.GetParameterList());
RCP<Matrix> xpetraA = Pr->BuildMatrix();
RCP<crs_matrix_type> A = MueLuUtilities::Op2NonConstTpetraCrs(xpetraA);
RCP<const driver_map_type> map = MueLuUtilities::Map2TpetraMap(*xpetraMap);
// ===================================================
// Domain Decomposition Preconditioner
// ===================================
//Creation of the MueLu list for the DD preconditioner
RCP<ParameterList> dd_list = rcp(new Teuchos::ParameterList());
dd_list->setName("MueLu");
dd_list->set("verbosity", "low");
dd_list->set("number of equations", 1);
dd_list->set("max levels", 1);
dd_list->set("coarse: type", "SCHWARZ"); //FOR A ONE LEVEL PRECONDITIONER THE COARSE LEVEL IS INTERPRETED AS SMOOTHING LEVEL
ParameterList& dd_smooth_sublist = dd_list->sublist("coarse: params");
dd_smooth_sublist.set("schwarz: overlap level", 0);
dd_smooth_sublist.set("schwarz: combine mode", "Zero");
dd_smooth_sublist.set("subdomain solver name", "RILUK");
ParameterList& coarse_subdomain_solver = dd_smooth_sublist.sublist("subdomain solver parameters");
coarse_subdomain_solver.set("fact: iluk level-of-fill", 3);
coarse_subdomain_solver.set("fact: absolute threshold", 0.);
coarse_subdomain_solver.set("fact: relative threshold", 1.);
coarse_subdomain_solver.set("fact: relax value", 0.);
RCP<muelu_tpetra_operator_type> B_DD = MueLu::CreateTpetraPreconditioner( (RCP<operator_type>)A, *dd_list );
// ===================================================
// Multi Grid Preconditioner
// ===================================
RCP<muelu_tpetra_operator_type> M;
M = MueLu::CreateTpetraPreconditioner( (RCP<operator_type>)A, mueluParams, Utilities::MV2NonConstTpetraMV(coordinates) );
RCP<multivector_type> X_muelu = rcp(new multivector_type(map,1));
RCP<multivector_type> B = rcp(new multivector_type(map,1));
RCP<linear_problem_type> Problem_muelu;
X_muelu->putScalar((scalar_type) 0.0);
B->randomize();
Problem_muelu = rcp(new linear_problem_type(A, X_muelu, B));
RCP<ParameterList> belosList = rcp(new ParameterList());
belosList->set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList->set("Convergence Tolerance", tol); // Relative convergence tolerance requested
//belosList->set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList->set("Verbosity", Belos::Errors);
belosList->set("Output Frequency", 1);
belosList->set("Output Style", Belos::Brief);
belosList->set("Implicit Residual Scaling", "None");
RCP<belos_solver_manager_type> solver;
if (krylovSolverType == "cg")
solver = rcp(new belos_pseudocg_manager_type(Problem_muelu, belosList));
else if (krylovSolverType == "gmres")
solver = rcp(new belos_gmres_manager_type(Problem_muelu, belosList));
else if (krylovSolverType == "bicgstab")
solver = rcp(new belos_bicgstab_manager_type(Problem_muelu, belosList));
else
throw std::invalid_argument("bad Krylov solver type");
for(int trial = 1; trial<=number_runs; ++trial)
{
X_muelu->putScalar((scalar_type) 0.0);
B->randomize();
//
// Set up Krylov solver and iterate.
//
Problem_muelu = rcp(new linear_problem_type(A, X_muelu, B));
Problem_muelu->setRightPrec(M);
Problem_muelu->setProblem();
solver->setProblem(Problem_muelu);
solver->solve();
int numIterations_muelu = solver->getNumIters();
Teuchos::Array<typename Teuchos::ScalarTraits<scalar_type>::magnitudeType> normVec_muelu(1);
multivector_type residual_muelu(B->getMap(),1);
A->apply(*X_muelu, residual_muelu);
residual_muelu.update(1.0, *B, -1.0);
residual_muelu.norm2(normVec_muelu);
if (mypid == 0) {
std::cout << "number of iterations with MueLu preconditioner= " << numIterations_muelu << std::endl;
std::cout << "||Residual|| = " << normVec_muelu[0] << std::endl;
}
}
#include <Teuchos_TimeMonitor.hpp>
Teuchos::TimeMonitor::summarize ();
return EXIT_SUCCESS;
}
void HierarchyUtils<Scalar, LocalOrdinal, GlobalOrdinal, Node>::AddNonSerializableDataToHierarchy(HierarchyManager& HM, Hierarchy& H, const ParameterList& paramList) {
for (ParameterList::ConstIterator it = paramList.begin(); it != paramList.end(); it++) {
const std::string& levelName = it->first;
// Check for mach of the form "level X" where X is a positive integer
if (paramList.isSublist(levelName) && levelName.find("level ") == 0 && levelName.size() > 6) {
int levelID = strtol(levelName.substr(6).c_str(), 0, 0);
if (levelID > 0)
{
// Do enough level adding so we can be sure to add the data to the right place
for (int i = H.GetNumLevels(); i <= levelID; i++)
H.AddNewLevel();
}
RCP<Level> level = H.GetLevel(levelID);
RCP<FactoryManager> M = Teuchos::rcp_dynamic_cast<FactoryManager>(HM.GetFactoryManager(levelID));
TEUCHOS_TEST_FOR_EXCEPTION(M.is_null(), Exceptions::InvalidArgument, "MueLu::Utils::AddNonSerializableDataToHierarchy: cannot get FactoryManager");
// Grab the level sublist & loop over parameters
const ParameterList& levelList = paramList.sublist(levelName);
for (ParameterList::ConstIterator it2 = levelList.begin(); it2 != levelList.end(); it2++) {
const std::string& name = it2->first;
TEUCHOS_TEST_FOR_EXCEPTION(name != "A" && name != "P" && name != "R" &&
name != "Nullspace" && name != "Coordinates" &&
!IsParamMuemexVariable(name), Exceptions::InvalidArgument,
"MueLu::Utils::AddNonSerializableDataToHierarchy: parameter list contains unknown data type");
if (name == "A") {
level->Set(name, Teuchos::getValue<RCP<Matrix > > (it2->second),NoFactory::get());
M->SetFactory(name, NoFactory::getRCP()); // TAW: not sure about this: be aware that this affects all levels
// However, A is accessible through NoFactory anyway, so it should
// be fine here.
}
else if( name == "P" || name == "R") {
level->AddKeepFlag(name,NoFactory::get(),MueLu::UserData);
level->Set(name, Teuchos::getValue<RCP<Matrix > > (it2->second), M->GetFactory(name).get());
}
else if (name == "Nullspace")
{
level->AddKeepFlag(name,NoFactory::get(),MueLu::UserData);
level->Set(name, Teuchos::getValue<RCP<MultiVector > >(it2->second), NoFactory::get());
//M->SetFactory(name, NoFactory::getRCP()); // TAW: generally it is a bad idea to overwrite the factory manager data here
// One should do this only in very special cases
}
else if(name == "Coordinates") //Scalar of Coordinates MV is always double
{
level->AddKeepFlag(name,NoFactory::get(),MueLu::UserData);
level->Set(name, Teuchos::getValue<RCP<Xpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> > >(it2->second), NoFactory::get());
//M->SetFactory(name, NoFactory::getRCP()); // TAW: generally it is a bad idea to overwrite the factory manager data here
}
#ifdef HAVE_MUELU_MATLAB
else
{
//Custom variable for Muemex
size_t typeNameStart = name.find_first_not_of(' ');
size_t typeNameEnd = name.find(' ', typeNameStart);
std::string typeName = name.substr(typeNameStart, typeNameEnd - typeNameStart);
std::transform(typeName.begin(), typeName.end(), typeName.begin(), ::tolower);
level->AddKeepFlag(name, NoFactory::get(), MueLu::UserData);
if(typeName == "matrix")
level->Set(name, Teuchos::getValue<RCP<Matrix> >(it2->second), NoFactory::get());
else if(typeName == "multivector")
level->Set(name, Teuchos::getValue<RCP<MultiVector> >(it2->second), NoFactory::get());
else if(typeName == "map")
level->Set(name, Teuchos::getValue<RCP<Xpetra::Map<LocalOrdinal, GlobalOrdinal, Node> > >(it2->second), NoFactory::get());
else if(typeName == "ordinalvector")
level->Set(name, Teuchos::getValue<RCP<Xpetra::Vector<LocalOrdinal, LocalOrdinal, GlobalOrdinal, Node> > >(it2->second), NoFactory::get());
else if(typeName == "scalar")
level->Set(name, Teuchos::getValue<Scalar>(it2->second), NoFactory::get());
else if(typeName == "double")
level->Set(name, Teuchos::getValue<double>(it2->second), NoFactory::get());
else if(typeName == "complex")
level->Set(name, Teuchos::getValue<std::complex<double> >(it2->second), NoFactory::get());
else if(typeName == "int")
level->Set(name, Teuchos::getValue<int>(it2->second), NoFactory::get());
else if(typeName == "string")
level->Set(name, Teuchos::getValue<std::string>(it2->second), NoFactory::get());
}
#endif
}
}
}
}
RCP<LinearOpWithSolveFactoryBase<double> >
LOWSFactoryBuilder::createLOWSFactory(const ParameterList& params)
{
/* check that we have a linear solver parameter list */
// TEST_FOR_EXCEPTION(params.name() != "Linear Solver",
// std::runtime_error,
// "Expected \"Linear Solver\" as name of parameter list input "
// "to createLOWSFactory()");
RCP<LinearOpWithSolveFactoryBase<double> > rtn;
RCP<PreconditionerFactoryBase<double> > prec;
if (params.isSublist("Amesos"))
{
RCP<ParameterList> p = rcp(new ParameterList(params.sublist("Amesos")));
rtn = rcp(new AmesosLinearOpWithSolveFactory());
rtn->setParameterList(p);
}
else if (params.isSublist("Aztec"))
{
RCP<ParameterList> p = rcp(new ParameterList(params.sublist("Aztec")));
rtn = rcp(new AztecOOLinearOpWithSolveFactory());
rtn->setParameterList(p);
}
else if (params.isSublist("Belos"))
{
RCP<ParameterList> p = rcp(new ParameterList(params.sublist("Belos")));
rtn = rcp(new BelosLinearOpWithSolveFactory<double>());
rtn->setParameterList(p);
}
else
{
TEST_FOR_EXCEPTION(true, std::runtime_error,
"solver parameter list did not contain one of [Aztec, Amesos, "
"Belos]");
}
if (params.isSublist("Preconditioner"))
{
ParameterList precParams = params.sublist("Preconditioner");
std::string precType = precParams.get<string>("Type");
if (precType=="ML")
{
std::string probType = getParameter<string>(precParams, "Problem Type");
ParameterList mlParams = precParams.sublist("ML Settings");
prec = rcp(new MLPreconditionerFactory(probType, mlParams));
}
else if (precType=="Ifpack")
{
std::string probType = getParameter<string>(precParams, "Prec Type");
RCP<ParameterList> ifpackParams
= rcp(new ParameterList(precParams.sublist("Ifpack")));
prec = rcp(new IfpackPreconditionerFactory());
prec->setParameterList(ifpackParams);
}
else
{
TEST_FOR_EXCEPTION(true, std::runtime_error,
"Preconditioner type [" << precType << "] not recognized");
}
}
TEST_FOR_EXCEPTION(prec.get() != 0 && !rtn->acceptsPreconditionerFactory(),
std::runtime_error,
"Huh? You have provided a preconditioner for a solver that cannot "
"accept a preconditioner!");
if (prec.get() != 0 && rtn->acceptsPreconditionerFactory())
{
rtn->setPreconditionerFactory(prec, "precond");
}
return rtn;
}
int main(int argc, char *argv[]) {
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
// initialize the random number generator
int ml_one = 1;
ML_srandom1(&ml_one);
// ===================== //
// create linear problem //
// ===================== //
ParameterList GaleriList;
GaleriList.set("nx", 10);
GaleriList.set("ny", 10);
GaleriList.set("nz", 10 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", 1);
GaleriList.set("mz", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);
Epetra_MultiVector* Coords = CreateCartesianCoordinates("3D",Map,GaleriList);
Epetra_Vector LHS(*Map);
Epetra_Vector RHS(*Map);
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
Teuchos::ParameterList MLList;
double TotalErrorResidual = 0.0, TotalErrorExactSol = 0.0;
// ====================== //
// default options for SA //
// ====================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type", "Gauss-Seidel");
char mystring[80];
strcpy(mystring,"SA");
TestMultiLevelPreconditioner(mystring, MLList, Problem,
TotalErrorResidual, TotalErrorExactSol);
// ============================================== //
// default options for SA, efficient symmetric GS //
// ============================================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type", "Gauss-Seidel");
MLList.set("smoother: Gauss-Seidel efficient symmetric",true);
TestMultiLevelPreconditioner(mystring, MLList, Problem,
TotalErrorResidual, TotalErrorExactSol,true);
// ============================== //
// default options for SA, Jacobi //
// ============================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type", "Jacobi");
TestMultiLevelPreconditioner(mystring, MLList, Problem, TotalErrorResidual,
TotalErrorExactSol,true);
// =========================== //
// default options for SA, Cheby //
// =========================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type", "Chebyshev");
TestMultiLevelPreconditioner(mystring, MLList, Problem,
TotalErrorResidual, TotalErrorExactSol);
// =========================== //
// Specifying Ifpack coarse lists correctly
// =========================== //
#ifdef HAVE_ML_IFPACK
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
if(!Comm.MyPID()) {
MLList.set("ML print initial list",1);
MLList.set("ML print final list",1);
}
MLList.set("smoother: type","ILU");
MLList.set("coarse: type","ILUT");
ParameterList &fList = MLList.sublist("smoother: ifpack list");
fList.set("fact: level-of-fill",1);
ParameterList &cList = MLList.sublist("coarse: ifpack list");
cList.set("fact: ilut level-of-fill",1e-2);
TestMultiLevelPreconditioner(mystring, MLList, Problem,
TotalErrorResidual, TotalErrorExactSol);
#endif
// =========================== //
// Specifying level sublists
// =========================== //
if (Comm.MyPID() == 0) PrintLine();
ParameterList LevelList;
ML_Epetra::SetDefaults("SA",LevelList);
ParameterList &smList = LevelList.sublist("smoother: list (level 0)");
smList.set("smoother: type","Jacobi");
smList.set("smoother: sweeps",5);
ParameterList &smList2 = LevelList.sublist("smoother: list (level 1)");
smList2.set("smoother: type","symmetric Gauss-Seidel");
smList2.set("smoother: sweeps",3);
ParameterList &coarseList = LevelList.sublist("coarse: list");
coarseList.set("smoother: type","symmetric Gauss-Seidel");
TestMultiLevelPreconditioner(mystring, LevelList, Problem,
TotalErrorResidual, TotalErrorExactSol);
// =========================== //
// Ifpack G-S w/ L1
// =========================== //
#ifdef HAVE_ML_IFPACK
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: use l1 Gauss-Seidel",true);
MLList.set("smoother: type", "Gauss-Seidel");
TestMultiLevelPreconditioner(mystring, MLList, Problem,
TotalErrorResidual, TotalErrorExactSol);
#endif
// =========================== //
// Ifpack SGS w/ L1
// =========================== //
#ifdef HAVE_ML_IFPACK
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: use l1 Gauss-Seidel",true);
MLList.set("smoother: type", "symmetric Gauss-Seidel");
TestMultiLevelPreconditioner(mystring, MLList, Problem,
TotalErrorResidual, TotalErrorExactSol);
#endif
// =========================== //
// Autodetected Line SGS (trivial lines)
// =========================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type", "line Gauss-Seidel");
MLList.set("smoother: line detection threshold",0.1);
MLList.set("x-coordinates",(*Coords)[0]);
MLList.set("y-coordinates",(*Coords)[1]);
MLList.set("z-coordinates",(*Coords)[2]);
TestMultiLevelPreconditioner(mystring, MLList, Problem,
TotalErrorResidual, TotalErrorExactSol);
// ===================== //
// print out total error //
// ===================== //
if (Comm.MyPID() == 0) {
cout << endl;
cout << "......Total error for residual = " << TotalErrorResidual << endl;
cout << "......Total error for exact solution = " << TotalErrorExactSol << endl;
cout << endl;
}
delete Matrix;
delete Coords;
delete Map;
if (TotalErrorResidual > 1e-8) {
cerr << "Error: `MultiLevelPrecoditioner_Sym.exe' failed!" << endl;
exit(EXIT_FAILURE);
}
#ifdef HAVE_MPI
MPI_Finalize();
#endif
if (Comm.MyPID() == 0)
cerr << "`MultiLevelPrecoditioner_Sym.exe' passed!" << endl;
return (EXIT_SUCCESS);
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
using Teuchos::ParameterList;
// =========================================================================
// MPI initialization using Teuchos
// =========================================================================
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// =========================================================================
// Convenient definitions
// =========================================================================
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
// =========================================================================
// Parameters initialization
// =========================================================================
Teuchos::CommandLineProcessor clp(false);
GO nx = 100, ny = 100, nz = 100;
Galeri::Xpetra::Parameters<GO> galeriParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file [default = 'scalingTest.xml']");
bool printTimings = true; clp.setOption("timings", "notimings", &printTimings, "print timings to screen");
int writeMatricesOPT = -2; clp.setOption("write", &writeMatricesOPT, "write matrices to file (-1 means all; i>=0 means level i)");
std::string dsolveType = "cg", solveType; clp.setOption("solver", &dsolveType, "solve type: (none | cg | gmres | standalone)");
double dtol = 1e-12, tol; clp.setOption("tol", &dtol, "solver convergence tolerance");
std::string mapFile; clp.setOption("map", &mapFile, "map data file");
std::string matrixFile; clp.setOption("matrix", &matrixFile, "matrix data file");
std::string coordFile; clp.setOption("coords", &coordFile, "coordinates data file");
int numRebuilds = 0; clp.setOption("rebuild", &numRebuilds, "#times to rebuild hierarchy");
int maxIts = 200; clp.setOption("its", &maxIts, "maximum number of solver iterations");
bool scaleResidualHistory = true; clp.setOption("scale", "noscale", &scaleResidualHistory, "scaled Krylov residual history");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
Xpetra::UnderlyingLib lib = xpetraParameters.GetLib();
ParameterList paramList;
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<ParameterList>(¶mList), *comm);
bool isDriver = paramList.isSublist("Run1");
if (isDriver) {
// update galeriParameters with the values from the XML file
ParameterList& realParams = galeriParameters.GetParameterList();
for (ParameterList::ConstIterator it = realParams.begin(); it != realParams.end(); it++) {
const std::string& name = realParams.name(it);
if (paramList.isParameter(name))
realParams.setEntry(name, paramList.getEntry(name));
}
}
// Retrieve matrix parameters (they may have been changed on the command line)
// [for instance, if we changed matrix type from 2D to 3D we need to update nz]
ParameterList galeriList = galeriParameters.GetParameterList();
// =========================================================================
// Problem construction
// =========================================================================
std::ostringstream galeriStream;
comm->barrier();
RCP<TimeMonitor> globalTimeMonitor = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
RCP<TimeMonitor> tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<Matrix> A;
RCP<const Map> map;
RCP<MultiVector> coordinates;
RCP<MultiVector> nullspace;
if (matrixFile.empty()) {
galeriStream << "========================================================\n" << xpetraParameters << galeriParameters;
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
std::string matrixType = galeriParameters.GetMatrixType();
// Create map and coordinates
// In the future, we hope to be able to first create a Galeri problem, and then request map and coordinates from it
// At the moment, however, things are fragile as we hope that the Problem uses same map and coordinates inside
if (matrixType == "Laplace1D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian1D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D", map, galeriList);
} else if (matrixType == "Laplace2D" || matrixType == "Star2D" ||
matrixType == "BigStar2D" || matrixType == "Elasticity2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D", map, galeriList);
} else if (matrixType == "Laplace3D" || matrixType == "Brick3D" || matrixType == "Elasticity3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D", map, galeriList);
}
// Expand map to do multiple DOF per node for block problems
if (matrixType == "Elasticity2D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 2);
if (matrixType == "Elasticity3D")
map = Xpetra::MapFactory<LO,GO,Node>::Build(map, 3);
galeriStream << "Processor subdomains in x direction: " << galeriList.get<int>("mx") << std::endl
<< "Processor subdomains in y direction: " << galeriList.get<int>("my") << std::endl
<< "Processor subdomains in z direction: " << galeriList.get<int>("mz") << std::endl
<< "========================================================" << std::endl;
if (matrixType == "Elasticity2D" || matrixType == "Elasticity3D") {
// Our default test case for elasticity: all boundaries of a square/cube have Neumann b.c. except left which has Dirichlet
galeriList.set("right boundary" , "Neumann");
galeriList.set("bottom boundary", "Neumann");
galeriList.set("top boundary" , "Neumann");
galeriList.set("front boundary" , "Neumann");
galeriList.set("back boundary" , "Neumann");
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(galeriParameters.GetMatrixType(), map, galeriList);
A = Pr->BuildMatrix();
nullspace = MultiVectorFactory::Build(map, 1);
if (matrixType == "Elasticity2D" ||
matrixType == "Elasticity3D") {
nullspace = Pr->BuildNullspace();
A->SetFixedBlockSize((galeriParameters.GetMatrixType() == "Elasticity2D") ? 2 : 3);
} else {
nullspace->putScalar(one);
}
} else {
if (!mapFile.empty())
map = Utils2::ReadMap(mapFile, xpetraParameters.GetLib(), comm);
comm->barrier();
if (lib == Xpetra::UseEpetra) {
A = Utils::Read(matrixFile, map);
} else {
// Tpetra matrix reader is still broken, so instead we read in
// a matrix in a binary format and then redistribute it
const bool binaryFormat = true;
A = Utils::Read(matrixFile, lib, comm, binaryFormat);
RCP<Matrix> newMatrix = MatrixFactory::Build(map, 1);
RCP<Import> importer = ImportFactory::Build(A->getRowMap(), map);
newMatrix->doImport(*A, *importer, Xpetra::INSERT);
newMatrix->fillComplete();
A.swap(newMatrix);
}
comm->barrier();
if (!coordFile.empty())
coordinates = Utils2::ReadMultiVector(coordFile, map);
nullspace = MultiVectorFactory::Build(map, 1);
nullspace->putScalar(one);
}
comm->barrier();
tm = Teuchos::null;
galeriStream << "Galeri complete.\n========================================================" << std::endl;
int numReruns = 1;
if (paramList.isParameter("number of reruns"))
numReruns = paramList.get<int>("number of reruns");
const bool mustAlreadyExist = true;
for (int rerunCount = 1; rerunCount <= numReruns; rerunCount++) {
ParameterList mueluList, runList;
bool stop = false;
if (isDriver) {
runList = paramList.sublist("Run1", mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} else {
mueluList = paramList;
stop = true;
}
int runCount = 1;
do {
A->SetMaxEigenvalueEstimate(-one);
solveType = dsolveType;
tol = dtol;
int savedOut = -1;
FILE* openedOut = NULL;
if (isDriver) {
if (runList.isParameter("filename")) {
// Redirect all output into a filename We have to redirect all output,
// including printf's, therefore we cannot simply replace C++ cout
// buffers, and have to use heavy machinary (dup2)
std::string filename = runList.get<std::string>("filename");
if (numReruns > 1)
filename += "_run" + MueLu::toString(rerunCount);
filename += (lib == Xpetra::UseEpetra ? ".epetra" : ".tpetra");
savedOut = dup(STDOUT_FILENO);
openedOut = fopen(filename.c_str(), "w");
dup2(fileno(openedOut), STDOUT_FILENO);
}
if (runList.isParameter("solver")) solveType = runList.get<std::string>("solver");
if (runList.isParameter("tol")) tol = runList.get<double> ("tol");
}
// Instead of checking each time for rank, create a rank 0 stream
RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::FancyOStream& fancyout = *fancy;
fancyout.setOutputToRootOnly(0);
fancyout << galeriStream.str();
// =========================================================================
// Preconditioner construction
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1.5 - MueLu read XML")));
RCP<HierarchyManager> mueLuFactory = rcp(new ParameterListInterpreter(mueluList));
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 2 - MueLu Setup")));
RCP<Hierarchy> H;
for (int i = 0; i <= numRebuilds; i++) {
A->SetMaxEigenvalueEstimate(-one);
H = mueLuFactory->CreateHierarchy();
H->GetLevel(0)->Set("A", A);
H->GetLevel(0)->Set("Nullspace", nullspace);
if (!coordinates.is_null())
H->GetLevel(0)->Set("Coordinates", coordinates);
mueLuFactory->SetupHierarchy(*H);
}
comm->barrier();
tm = Teuchos::null;
// =========================================================================
// System solution (Ax = b)
// =========================================================================
comm->barrier();
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3 - LHS and RHS initialization")));
RCP<Vector> X = VectorFactory::Build(map);
RCP<Vector> B = VectorFactory::Build(map);
{
// we set seed for reproducibility
Utils::SetRandomSeed(*comm);
X->randomize();
A->apply(*X, *B, Teuchos::NO_TRANS, one, zero);
Teuchos::Array<STS::magnitudeType> norms(1);
B->norm2(norms);
B->scale(one/norms[0]);
X->putScalar(zero);
}
tm = Teuchos::null;
if (writeMatricesOPT > -2) {
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 3.5 - Matrix output")));
H->Write(writeMatricesOPT, writeMatricesOPT);
tm = Teuchos::null;
}
comm->barrier();
if (solveType == "none") {
// Do not perform a solve
} else if (solveType == "standalone") {
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 4 - Fixed Point Solve")));
H->IsPreconditioner(false);
H->Iterate(*B, *X, maxIts);
} else if (solveType == "cg" || solveType == "gmres") {
#ifdef HAVE_MUELU_BELOS
tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 5 - Belos Solve")));
// Operator and Multivector type that will be used with Belos
typedef MultiVector MV;
typedef Belos::OperatorT<MV> OP;
H->IsPreconditioner(true);
// Define Operator and Preconditioner
Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO, LMO>(A)); // Turns a Xpetra::Matrix object into a Belos operator
Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp <SC, LO, GO, NO, LMO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
// Construct a Belos LinearProblem object
RCP< Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
belosProblem->setRightPrec(belosPrec);
bool set = belosProblem->setProblem();
if (set == false) {
fancyout << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
return EXIT_FAILURE;
}
// Belos parameter list
Teuchos::ParameterList belosList;
belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
belosList.set("Convergence Tolerance", tol); // Relative convergence tolerance requested
belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", Belos::Brief);
if (!scaleResidualHistory)
belosList.set("Implicit Residual Scaling", "None");
// Create an iterative solver manager
RCP< Belos::SolverManager<SC, MV, OP> > solver;
if (solveType == "cg") {
solver = rcp(new Belos::PseudoBlockCGSolMgr <SC, MV, OP>(belosProblem, rcp(&belosList, false)));
} else if (solveType == "gmres") {
solver = rcp(new Belos::BlockGmresSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
}
// Perform solve
Belos::ReturnType ret = Belos::Unconverged;
try {
ret = solver->solve();
// Get the number of iterations for this solve.
fancyout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
} catch(...) {
fancyout << std::endl << "ERROR: Belos threw an error! " << std::endl;
}
// Check convergence
if (ret != Belos::Converged)
fancyout << std::endl << "ERROR: Belos did not converge! " << std::endl;
else
fancyout << std::endl << "SUCCESS: Belos converged!" << std::endl;
#endif //ifdef HAVE_MUELU_BELOS
} else {
throw MueLu::Exceptions::RuntimeError("Unknown solver type: \"" + solveType + "\"");
}
comm->barrier();
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize(A->getRowMap()->getComm().ptr(), std::cout, false, true, false, Teuchos::Union);
TimeMonitor::clearCounters();
if (isDriver) {
if (openedOut != NULL) {
dup2(savedOut, STDOUT_FILENO);
fclose(openedOut);
openedOut = NULL;
}
try {
runList = paramList.sublist("Run" + MueLu::toString(++runCount), mustAlreadyExist);
mueluList = runList .sublist("MueLu", mustAlreadyExist);
} catch (std::exception) {
stop = true;
}
}
} while (stop == false);
}
return 0;
} //main
// compare metrics
bool ComparisonHelper::CompareMetrics(const ParameterList &metricsPlist, const RCP<const Comm<int> > &comm)
{
int rank = comm->getRank();
//get sources for problema nd reference
const string prb_name = metricsPlist.get<string>("Problem");
const string ref_name = metricsPlist.get<string>("Reference");
if(rank == 0) {
cout << "\nMetric/Timer comparison of: " << prb_name << " and ";
cout << ref_name <<" (reference source)\n";
}
// get sources
RCP<const ComparisonSource> sourcePrb = this->sources[prb_name];
RCP<const ComparisonSource> sourceRef = this->sources[ref_name];
// get timing data
std::map< const string, const double> prb_timers = this->timerDataToMap(sourcePrb->timers);
std::map< const string, const double> ref_timers = this->timerDataToMap(sourceRef->timers);
// get all of the metrics to be tested
std::queue<ParameterList> metrics = ComparisonHelper::getMetricsToCompare(metricsPlist);
// run comparison
int all_tests_pass = 1;
string metric_name;
while(!metrics.empty()) {
// print their names...
ostringstream msg;
metric_name = metrics.front().name();
if (metric_name == "Metrics") { // special key word means compare the metrics list
std::vector<MetricAnalyzerInfo> metricInfoSetPrb;
std::vector<MetricAnalyzerInfo> metricInfoSetRef;
MetricAnalyzer::LoadMetricInfo(metricInfoSetPrb, sourcePrb.get()->metricObject, metricsPlist.sublist("Metrics"));
MetricAnalyzer::LoadMetricInfo(metricInfoSetRef, sourceRef.get()->metricObject, metricsPlist.sublist("Metrics"));
// there is some redundancy here because the metric info holds both the questions and the results
// this happened because I wanted to reuse the MetricAnalyzer code for loading metric checks or comparisons
// we can iterate over either to get the questions
for (size_t n = 0; n < metricInfoSetPrb.size(); ++n) {
if(!ComparisonHelper::metricComparisonTest(comm, metricInfoSetPrb[n], metricInfoSetRef[n], metrics.front(), msg)) {
all_tests_pass = 0;
}
if(rank == 0) {
cout << msg.str() << endl;
}
}
}
else if(prb_timers.find(metric_name) != prb_timers.end() && ref_timers.find(metric_name) != ref_timers.end()) {
if(rank == 0) cout << "\ncomparing timer: " << metric_name << endl;
if(!ComparisonHelper::timerComparisonTest(comm,
prb_timers.at(metric_name),
ref_timers.at(metric_name),
metrics.front(), msg)) {
all_tests_pass = 0;
if (rank == 0) {
cout << "timer comparison test caused a FAILED event." << endl;
}
}
if(rank == 0) {
cout << msg.str() << endl;
}
}
metrics.pop();
}
if(rank == 0) {
if(all_tests_pass == 1) {
cout << "\nAll metric/timer comparisons PASSED." << endl;
}
else {
cout << "\nMetric/timer metric comparisons FAILED." << endl;
}
}
return (all_tests_pass == 1);
}
Teuchos::RCP<MueLu::TpetraOperator<Scalar,LocalOrdinal,GlobalOrdinal,Node> >
CreateTpetraPreconditioner(const Teuchos::RCP<Tpetra::Operator<Scalar, LocalOrdinal, GlobalOrdinal, Node> > &inA,
Teuchos::ParameterList& inParamList,
const Teuchos::RCP<Tpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node>>& inCoords = Teuchos::null,
const Teuchos::RCP<Tpetra::MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>>& inNullspace = Teuchos::null)
{
typedef Scalar SC;
typedef LocalOrdinal LO;
typedef GlobalOrdinal GO;
typedef Node NO;
using Teuchos::ParameterList;
typedef Xpetra::MultiVector<SC,LO,GO,NO> MultiVector;
typedef Xpetra::Matrix<SC,LO,GO,NO> Matrix;
typedef Hierarchy<SC,LO,GO,NO> Hierarchy;
typedef HierarchyManager<SC,LO,GO,NO> HierarchyManager;
typedef Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> crs_matrix_type;
typedef Tpetra::Experimental::BlockCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> block_crs_matrix_type;
bool hasParamList = inParamList.numParams();
RCP<HierarchyManager> mueLuFactory;
ParameterList paramList = inParamList;
RCP<const crs_matrix_type> constCrsA;
RCP<crs_matrix_type> crsA;
#if defined(HAVE_MUELU_EXPERIMENTAL) and defined(HAVE_MUELU_AMGX)
std::string externalMG = "use external multigrid package";
if (hasParamList && paramList.isParameter(externalMG) && paramList.get<std::string>(externalMG) == "amgx"){
constCrsA = rcp_dynamic_cast<const crs_matrix_type>(inA);
TEUCHOS_TEST_FOR_EXCEPTION(constCrsA == Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: failed to dynamic cast to Tpetra::CrsMatrix, which is required to be able to use AmgX.");
return rcp(new AMGXOperator<SC,LO,GO,NO>(inA,inParamList));
}
#endif
std::string syntaxStr = "parameterlist: syntax";
if (hasParamList && paramList.isParameter(syntaxStr) && paramList.get<std::string>(syntaxStr) == "ml") {
paramList.remove(syntaxStr);
mueLuFactory = rcp(new MLParameterListInterpreter<SC,LO,GO,NO>(paramList));
} else {
mueLuFactory = rcp(new ParameterListInterpreter <SC,LO,GO,NO>(paramList,inA->getDomainMap()->getComm()));
}
RCP<Hierarchy> H = mueLuFactory->CreateHierarchy();
H->setlib(Xpetra::UseTpetra);
// Wrap A
RCP<Matrix> A;
RCP<block_crs_matrix_type> bcrsA = rcp_dynamic_cast<block_crs_matrix_type>(inA);
crsA = rcp_dynamic_cast<crs_matrix_type>(inA);
if (crsA != Teuchos::null)
A = TpetraCrs_To_XpetraMatrix<SC,LO,GO,NO>(crsA);
else if (bcrsA != Teuchos::null) {
RCP<Xpetra::CrsMatrix<SC,LO,GO,NO> > temp = rcp(new Xpetra::TpetraBlockCrsMatrix<SC,LO,GO,NO>(bcrsA));
TEUCHOS_TEST_FOR_EXCEPTION(temp==Teuchos::null, Exceptions::RuntimeError, "CreateTpetraPreconditioner: cast from Tpetra::Experimental::BlockCrsMatrix to Xpetra::TpetraBlockCrsMatrix failed.");
A = rcp(new Xpetra::CrsMatrixWrap<SC,LO,GO,NO>(temp));
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "CreateTpetraPreconditioner: only Tpetra CrsMatrix and BlockCrsMatrix types are supported.");
}
H->GetLevel(0)->Set("A", A);
// Wrap coordinates if available
if (inCoords != Teuchos::null) {
RCP<Xpetra::MultiVector<double,LO,GO,NO> > coordinates = TpetraMultiVector_To_XpetraMultiVector<double,LO,GO,NO>(inCoords);
H->GetLevel(0)->Set("Coordinates", coordinates);
}
// Wrap nullspace if available, otherwise use constants
RCP<MultiVector> nullspace;
if (inNullspace != Teuchos::null) {
nullspace = TpetraMultiVector_To_XpetraMultiVector<SC,LO,GO,NO>(inNullspace);
} else {
int nPDE = MasterList::getDefault<int>("number of equations");
if (paramList.isSublist("Matrix")) {
// Factory style parameter list
const Teuchos::ParameterList& operatorList = paramList.sublist("Matrix");
if (operatorList.isParameter("PDE equations"))
nPDE = operatorList.get<int>("PDE equations");
} else if (paramList.isParameter("number of equations")) {
// Easy style parameter list
nPDE = paramList.get<int>("number of equations");
}
nullspace = Xpetra::MultiVectorFactory<SC,LO,GO,NO>::Build(A->getDomainMap(), nPDE);
if (nPDE == 1) {
nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
} else {
for (int i = 0; i < nPDE; i++) {
Teuchos::ArrayRCP<SC> nsData = nullspace->getDataNonConst(i);
for (int j = 0; j < nsData.size(); j++) {
GO GID = A->getDomainMap()->getGlobalElement(j) - A->getDomainMap()->getIndexBase();
if ((GID-i) % nPDE == 0)
nsData[j] = Teuchos::ScalarTraits<SC>::one();
}
}
}
}
H->GetLevel(0)->Set("Nullspace", nullspace);
Teuchos::ParameterList nonSerialList,dummyList;
ExtractNonSerializableData(paramList, dummyList, nonSerialList);
HierarchyUtils<SC,LO,GO,NO>::AddNonSerializableDataToHierarchy(*mueLuFactory,*H, nonSerialList);
mueLuFactory->SetupHierarchy(*H);
return rcp(new TpetraOperator<SC,LO,GO,NO>(H));
}
void ParameterListInterpreter<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::SetEasyParameterList(const Teuchos::ParameterList& constParamList) {
// Create a non const copy of the parameter list
// Working with a modifiable list is much much easier than with original one
ParameterList paramList = constParamList;
// Translate cycle type parameter
if (paramList.isParameter("cycle type")) {
std::map<std::string,CycleType> cycleMap;
cycleMap["V"] = VCYCLE;
cycleMap["W"] = WCYCLE;
std::string cycleType = paramList.get<std::string>("cycle type");
TEUCHOS_TEST_FOR_EXCEPTION(cycleMap.count(cycleType) == 0, Exceptions::RuntimeError, "Invalid cycle type: \"" << cycleType << "\"");
Cycle_ = cycleMap[cycleType];
}
this->maxCoarseSize_ = paramList.get<int> ("coarse: max size", Hierarchy::GetDefaultMaxCoarseSize());
this->numDesiredLevel_ = paramList.get<int> ("max levels", Hierarchy::GetDefaultMaxLevels());
this->graphOutputLevel_ = paramList.get<int> ("debug: graph level", -1);
blockSize_ = paramList.get<int> ("number of equations", 1);
// Save level data
if (paramList.isSublist("print")) {
ParameterList printList = paramList.sublist("print");
if (printList.isParameter("A"))
this->matricesToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "A");
if (printList.isParameter("P"))
this->prolongatorsToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "P");
if (printList.isParameter("R"))
this->restrictorsToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "R");
}
// Translate verbosity parameter
this->verbosity_ = static_cast<MsgType>(Hierarchy::GetDefaultVerbLevel()); // cast int to enum
if (paramList.isParameter("verbosity")) {
std::map<std::string,MsgType> verbMap;
verbMap["none"] = None;
verbMap["low"] = Low;
verbMap["medium"] = Medium;
verbMap["high"] = High;
verbMap["extreme"] = Extreme;
verbMap["test"] = Test;
std::string verbosityLevel = paramList.get<std::string>("verbosity");
TEUCHOS_TEST_FOR_EXCEPTION(verbMap.count(verbosityLevel) == 0, Exceptions::RuntimeError, "Invalid verbosity level: \"" << verbosityLevel << "\"");
this->verbosity_ = verbMap[verbosityLevel];
this->SetVerbLevel(this->verbosity_);
}
// Detect if we need to transfer coordinates to coarse levels. We do that iff
// - we use "laplacian" dropping on some level, or
// - we use repartitioning on some level
// This is not ideal, as we may have "repartition: enable" turned on by default
// and not present in the list, but it is better than nothing.
useCoordinates_ = false;
if ((paramList.isParameter("repartition: enable") && paramList.get<bool>("repartition: enable") == true) ||
(paramList.isParameter("aggregation: drop scheme") && paramList.get<std::string>("aggregation: drop scheme") == "laplacian")) {
useCoordinates_ = true;
} else {
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
std::string levelStr = "level" + toString(levelID);
if (paramList.isSublist(levelStr)) {
const ParameterList& levelList = paramList.sublist(levelStr);
if ((levelList.isParameter("repartition: enable") && levelList.get<bool>("repartition: enable") == true) ||
(levelList.isParameter("aggregation: drop scheme") && levelList.get<std::string>("aggregation: drop scheme") == "laplacian")) {
useCoordinates_ = true;
break;
}
}
}
}
// Detect if we do implicit P and R rebalance
if (paramList.isParameter("repartition: enable") && paramList.get<bool>("repartition: enable") == true)
this->doPRrebalance_ = paramList.get<bool>("repartition: rebalance P and R", Hierarchy::GetDefaultPRrebalance());
this->implicitTranspose_ = paramList.get<bool>("transpose: use implicit", Hierarchy::GetDefaultImplicitTranspose());
// Create default manager
RCP<FactoryManager> defaultManager = rcp(new FactoryManager());
defaultManager->SetVerbLevel(this->verbosity_);
UpdateFactoryManager(paramList, ParameterList(), *defaultManager);
defaultManager->Print();
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
RCP<FactoryManager> levelManager;
if (paramList.isSublist("level " + toString(levelID))) {
// Some level specific parameters, update default manager
bool mustAlreadyExist = true;
ParameterList& levelList = paramList.sublist("level " + toString(levelID), mustAlreadyExist);
levelManager = rcp(new FactoryManager(*defaultManager));
levelManager->SetVerbLevel(defaultManager->GetVerbLevel());
UpdateFactoryManager(levelList, paramList, *levelManager);
} else {
// No level specific parameter, use default manager
levelManager = defaultManager;
}
this->AddFactoryManager(levelID, 1, levelManager);
}
if (paramList.isParameter("strict parameter checking") &&
paramList.get<bool> ("strict parameter checking")) {
ParameterList unusedParamList;
// Check for unused parameters that aren't lists
for (ParameterList::ConstIterator itr = paramList.begin(); itr != paramList.end(); ++itr) {
const ParameterEntry& entry = paramList.entry(itr);
if (!entry.isList() && !entry.isUsed())
unusedParamList.setEntry(paramList.name(itr), entry);
}
#if 0
// Check for unused parameters in level-specific sublists
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
std::string levelStr = "level" + toString(levelID);
if (paramList.isSublist(levelStr)) {
const ParameterList& levelList = paramList.sublist(levelStr);
for (ParameterList::ConstIterator itr = levelList.begin(); itr != levelList.end(); ++itr) {
const ParameterEntry& entry = levelList.entry(itr);
if (!entry.isList() && !entry.isUsed())
unusedParamList.sublist(levelStr).setEntry(levelList.name(itr), entry);
}
}
}
#endif
if (unusedParamList.numParams() > 0) {
std::ostringstream unusedParamsStream;
int indent = 4;
unusedParamList.print(unusedParamsStream, indent);
TEUCHOS_TEST_FOR_EXCEPTION_PURE_MSG(true, Teuchos::Exceptions::InvalidParameter,
"WARNING: Unused parameters were detected. Please check spelling and type." << std::endl << unusedParamsStream.str());
}
}
// FIXME: parameters passed to packages, like Ifpack2, are not touched by us, resulting in "[unused]" flag
// being displayed. On the other hand, we don't want to simply iterate through them touching. I don't know
// what a good solution looks like
this->GetOStream(static_cast<MsgType>(Runtime1 | Test), 0) << paramList << std::endl;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
int mypid = Comm.MyPID();
#else
Epetra_SerialComm Comm;
int mypid = 0;
#endif
// Read XML input deck
ParameterList masterList;
if (argc > 1) {
if (strncmp("-h",argv[1],2) == 0) {
cout << "help" << endl;
ML_Print_Help();
ML_Exit(mypid,0,EXIT_SUCCESS);
}
else {
int i=0,j;
FILE* fid = fopen(argv[1],"r");
if (fid) {
i++;
fclose(fid);
}
Comm.SumAll(&i, &j, 1);
if (j!=Comm.NumProc()) {
cout << "Could not open input file." << endl;
ML_Print_Help();
ML_Exit(mypid,0,EXIT_FAILURE);
}
FileInputSource fileSrc(argv[1]);
XMLObject fileXML = fileSrc.getObject();
XMLParameterListReader ListReader;
masterList = ListReader.toParameterList(fileXML);
}
} else {
cout << "No input file specified." << endl;
ML_Print_Help();
ML_Exit(mypid,0,EXIT_SUCCESS);
}
ParameterList *fileList, *AztecOOList;
try {fileList = &(masterList.sublist("data files",true));}
catch(...) {ML_Exit(mypid,"Missing \"data files\" sublist.",EXIT_FAILURE);}
try {AztecOOList = &(masterList.sublist("AztecOO"));}
catch(...) {ML_Exit(mypid,"Missing \"AztecOO\" sublist.",EXIT_FAILURE);}
#ifdef ML_SCALING
const int ntimers=4;
enum {total, probBuild, precBuild, solve};
ml_DblLoc timeVec[ntimers], maxTime[ntimers], minTime[ntimers];
for (int i=0; i<ntimers; i++) timeVec[i].rank = Comm.MyPID();
timeVec[total].value = MPI_Wtime();
#endif
string matrixfile = fileList->get("matrix input file","A.dat");
const char *datafile = matrixfile.c_str();
int numGlobalRows;
ML_Read_Matrix_Dimensions(datafile, &numGlobalRows, Comm);
#ifdef ML_SCALING
timeVec[probBuild].value = MPI_Wtime();
#endif
// ===================================================== //
// READ IN MATRICES FROM FILE //
// ===================================================== //
if (!mypid) printf("reading %s\n",datafile); fflush(stdout);
Epetra_CrsMatrix *Amat=NULL;
//Epetra_Map *RowMap=NULL;
int errCode=0;
//if (RowMap) errCode=EpetraExt::MatrixMarketFileToCrsMatrix(datafile, *RowMap, Amat);
//else errCode=EpetraExt::MatrixMarketFileToCrsMatrix(datafile, Comm, Amat);
errCode=EpetraExt::MatrixMarketFileToCrsMatrix(datafile, Comm, Amat);
if (errCode) ML_Exit(mypid,"error reading matrix", EXIT_FAILURE);
Amat->OptimizeStorage();
Epetra_Vector LHS(Amat->RowMap()); LHS.Random();
Epetra_Vector RHS(Amat->RowMap()); RHS.PutScalar(0.0);
Epetra_LinearProblem Problem(Amat, &LHS, &RHS);
#ifdef ML_SCALING
timeVec[probBuild].value = MPI_Wtime() - timeVec[probBuild].value;
#endif
// =========================== build preconditioner ===========================
#ifdef ML_SCALING
timeVec[precBuild].value = MPI_Wtime();
#endif
// no preconditioner right now
#ifdef ML_SCALING
timeVec[precBuild].value = MPI_Wtime() - timeVec[precBuild].value;
#endif
// =========================== outer solver =============================
#ifdef ML_SCALING
timeVec[solve].value = MPI_Wtime();
#endif
solver.SetParameters(*AztecOOList);
int maxits = AztecOOList->get("Aztec iterations",250);
double tol = AztecOOList->get("Aztec tolerance",1e-10);
#ifdef ML_SCALING
timeVec[solve].value = MPI_Wtime() - timeVec[solve].value;
#endif
// compute the real residual
double residual;
LHS.Norm2(&residual);
if( Comm.MyPID()==0 ) {
cout << "||b-Ax||_2 = " << residual << endl;
}
delete Amat;
//delete RowMap;
#ifdef ML_SCALING
timeVec[total].value = MPI_Wtime() - timeVec[total].value;
//avg
double dupTime[ntimers],avgTime[ntimers];
for (int i=0; i<ntimers; i++) dupTime[i] = timeVec[i].value;
MPI_Reduce(dupTime,avgTime,ntimers,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
for (int i=0; i<ntimers; i++) avgTime[i] = avgTime[i]/Comm.NumProc();
//min
MPI_Reduce(timeVec,minTime,ntimers,MPI_DOUBLE_INT,MPI_MINLOC,0,MPI_COMM_WORLD);
//max
MPI_Reduce(timeVec,maxTime,ntimers,MPI_DOUBLE_INT,MPI_MAXLOC,0,MPI_COMM_WORLD);
if (Comm.MyPID() == 0) {
printf("timing : max (pid) min (pid) avg\n");
printf("Problem build : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[probBuild].value,maxTime[probBuild].rank,
minTime[probBuild].value,minTime[probBuild].rank,
avgTime[probBuild]);
printf("Preconditioner build : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[precBuild].value,maxTime[precBuild].rank,
minTime[precBuild].value,minTime[precBuild].rank,
avgTime[precBuild]);
printf("Solve : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[solve].value,maxTime[solve].rank,
minTime[solve].value,minTime[solve].rank,
avgTime[solve]);
printf("Total : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[total].value,maxTime[total].rank,
minTime[total].value,minTime[total].rank,
avgTime[total]);
}
#endif
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} //main
