TEUCHOS_UNIT_TEST(Teuchos_ParameterList, parameterEntryXMLConverters)
{
ParameterList myList;
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(int, 2);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(unsigned int, 3);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(short int, 4);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(unsigned short int, 5);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(long int, 6);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(unsigned long int, 7);
#ifdef HAVE_TEUCHOS_LONG_LONG_INT
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(long long int, 8);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(unsigned long long int, 9);
#endif //HAVE_TEUCHOS_LONG_LONG_INT
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(double, 10.0);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(float, 11.0);
ADD_TYPE_AND_ARRAY_TYPE_PARAMETER(std::string, "hello");
ADD_TYPE_PARAMETER(char, 'a');
ADD_TYPE_PARAMETER(bool, true);
RCP<ParameterList> readInPL = writeThenReadPL(myList);
out << "\nmyList:\n";
myList.print(out);
out << "\n*readInPL:\n";
readInPL->print(out);
TEST_ASSERT(haveSameValues(myList, *readInPL));
}
void ParameterListInterpreter<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::SetEasyParameterList(const Teuchos::ParameterList& constParamList) {
// Create a non const copy of the parameter list
// Working with a modifiable list is much much easier than with original one
ParameterList paramList = constParamList;
// Translate cycle type parameter
if (paramList.isParameter("cycle type")) {
std::map<std::string,CycleType> cycleMap;
cycleMap["V"] = VCYCLE;
cycleMap["W"] = WCYCLE;
std::string cycleType = paramList.get<std::string>("cycle type");
TEUCHOS_TEST_FOR_EXCEPTION(cycleMap.count(cycleType) == 0, Exceptions::RuntimeError, "Invalid cycle type: \"" << cycleType << "\"");
Cycle_ = cycleMap[cycleType];
}
this->maxCoarseSize_ = paramList.get<int> ("coarse: max size", Hierarchy::GetDefaultMaxCoarseSize());
this->numDesiredLevel_ = paramList.get<int> ("max levels", Hierarchy::GetDefaultMaxLevels());
this->graphOutputLevel_ = paramList.get<int> ("debug: graph level", -1);
blockSize_ = paramList.get<int> ("number of equations", 1);
// Save level data
if (paramList.isSublist("print")) {
ParameterList printList = paramList.sublist("print");
if (printList.isParameter("A"))
this->matricesToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "A");
if (printList.isParameter("P"))
this->prolongatorsToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "P");
if (printList.isParameter("R"))
this->restrictorsToPrint_ = Teuchos::getArrayFromStringParameter<int>(printList, "R");
}
// Translate verbosity parameter
this->verbosity_ = static_cast<MsgType>(Hierarchy::GetDefaultVerbLevel()); // cast int to enum
if (paramList.isParameter("verbosity")) {
std::map<std::string,MsgType> verbMap;
verbMap["none"] = None;
verbMap["low"] = Low;
verbMap["medium"] = Medium;
verbMap["high"] = High;
verbMap["extreme"] = Extreme;
verbMap["test"] = Test;
std::string verbosityLevel = paramList.get<std::string>("verbosity");
TEUCHOS_TEST_FOR_EXCEPTION(verbMap.count(verbosityLevel) == 0, Exceptions::RuntimeError, "Invalid verbosity level: \"" << verbosityLevel << "\"");
this->verbosity_ = verbMap[verbosityLevel];
this->SetVerbLevel(this->verbosity_);
}
// Detect if we need to transfer coordinates to coarse levels. We do that iff
// - we use "laplacian" dropping on some level, or
// - we use repartitioning on some level
// This is not ideal, as we may have "repartition: enable" turned on by default
// and not present in the list, but it is better than nothing.
useCoordinates_ = false;
if ((paramList.isParameter("repartition: enable") && paramList.get<bool>("repartition: enable") == true) ||
(paramList.isParameter("aggregation: drop scheme") && paramList.get<std::string>("aggregation: drop scheme") == "laplacian")) {
useCoordinates_ = true;
} else {
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
std::string levelStr = "level" + toString(levelID);
if (paramList.isSublist(levelStr)) {
const ParameterList& levelList = paramList.sublist(levelStr);
if ((levelList.isParameter("repartition: enable") && levelList.get<bool>("repartition: enable") == true) ||
(levelList.isParameter("aggregation: drop scheme") && levelList.get<std::string>("aggregation: drop scheme") == "laplacian")) {
useCoordinates_ = true;
break;
}
}
}
}
// Detect if we do implicit P and R rebalance
if (paramList.isParameter("repartition: enable") && paramList.get<bool>("repartition: enable") == true)
this->doPRrebalance_ = paramList.get<bool>("repartition: rebalance P and R", Hierarchy::GetDefaultPRrebalance());
this->implicitTranspose_ = paramList.get<bool>("transpose: use implicit", Hierarchy::GetDefaultImplicitTranspose());
// Create default manager
RCP<FactoryManager> defaultManager = rcp(new FactoryManager());
defaultManager->SetVerbLevel(this->verbosity_);
UpdateFactoryManager(paramList, ParameterList(), *defaultManager);
defaultManager->Print();
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
RCP<FactoryManager> levelManager;
if (paramList.isSublist("level " + toString(levelID))) {
// Some level specific parameters, update default manager
bool mustAlreadyExist = true;
ParameterList& levelList = paramList.sublist("level " + toString(levelID), mustAlreadyExist);
levelManager = rcp(new FactoryManager(*defaultManager));
levelManager->SetVerbLevel(defaultManager->GetVerbLevel());
UpdateFactoryManager(levelList, paramList, *levelManager);
} else {
// No level specific parameter, use default manager
levelManager = defaultManager;
}
this->AddFactoryManager(levelID, 1, levelManager);
}
if (paramList.isParameter("strict parameter checking") &&
paramList.get<bool> ("strict parameter checking")) {
ParameterList unusedParamList;
// Check for unused parameters that aren't lists
for (ParameterList::ConstIterator itr = paramList.begin(); itr != paramList.end(); ++itr) {
const ParameterEntry& entry = paramList.entry(itr);
if (!entry.isList() && !entry.isUsed())
unusedParamList.setEntry(paramList.name(itr), entry);
}
#if 0
// Check for unused parameters in level-specific sublists
for (int levelID = 0; levelID < this->numDesiredLevel_; levelID++) {
std::string levelStr = "level" + toString(levelID);
if (paramList.isSublist(levelStr)) {
const ParameterList& levelList = paramList.sublist(levelStr);
for (ParameterList::ConstIterator itr = levelList.begin(); itr != levelList.end(); ++itr) {
const ParameterEntry& entry = levelList.entry(itr);
if (!entry.isList() && !entry.isUsed())
unusedParamList.sublist(levelStr).setEntry(levelList.name(itr), entry);
}
}
}
#endif
if (unusedParamList.numParams() > 0) {
std::ostringstream unusedParamsStream;
int indent = 4;
unusedParamList.print(unusedParamsStream, indent);
TEUCHOS_TEST_FOR_EXCEPTION_PURE_MSG(true, Teuchos::Exceptions::InvalidParameter,
"WARNING: Unused parameters were detected. Please check spelling and type." << std::endl << unusedParamsStream.str());
}
}
// FIXME: parameters passed to packages, like Ifpack2, are not touched by us, resulting in "[unused]" flag
// being displayed. On the other hand, we don't want to simply iterate through them touching. I don't know
// what a good solution looks like
this->GetOStream(static_cast<MsgType>(Runtime1 | Test), 0) << paramList << std::endl;
}
int main(int narg, char *arg[]) {
Teuchos::GlobalMPISession mpiSession(&narg, &arg,0);
Platform &platform = Tpetra::DefaultPlatform::getDefaultPlatform();
RCP<const Teuchos::Comm<int> > CommT = platform.getComm();
int me = CommT->getRank();
int numProcs = CommT->getSize();
if (me == 0){
cout
<< "====================================================================\n"
<< "| |\n"
<< "| Example: Partition Pamgen Hexahedral Mesh |\n"
<< "| |\n"
<< "| Questions? Contact Karen Devine ([email protected]), |\n"
<< "| Erik Boman ([email protected]), |\n"
<< "| Siva Rajamanickam ([email protected]). |\n"
<< "| |\n"
<< "| Pamgen's website: http://trilinos.sandia.gov/packages/pamgen |\n"
<< "| Zoltan2's website: http://trilinos.sandia.gov/packages/zoltan2 |\n"
<< "| Trilinos website: http://trilinos.sandia.gov |\n"
<< "| |\n"
<< "====================================================================\n";
}
#ifdef HAVE_MPI
if (me == 0) {
cout << "PARALLEL executable \n";
}
#else
if (me == 0) {
cout << "SERIAL executable \n";
}
#endif
/***************************************************************************/
/*************************** GET XML INPUTS ********************************/
/***************************************************************************/
// default values for command-line arguments
std::string xmlMeshInFileName("Poisson.xml");
std::string action("mj");
int nParts = CommT->getSize();
// Read run-time options.
Teuchos::CommandLineProcessor cmdp (false, false);
cmdp.setOption("xmlfile", &xmlMeshInFileName,
"XML file with PamGen specifications");
cmdp.setOption("action", &action,
"Method to use: mj, scotch, zoltan_rcb, zoltan_hg, hg_ghost, "
"parma or color");
cmdp.setOption("nparts", &nParts,
"Number of parts to create");
cmdp.parse(narg, arg);
// Read xml file into parameter list
ParameterList inputMeshList;
if(xmlMeshInFileName.length()) {
if (me == 0) {
cout << "\nReading parameter list from the XML file \""
<<xmlMeshInFileName<<"\" ...\n\n";
}
Teuchos::updateParametersFromXmlFile(xmlMeshInFileName,
Teuchos::inoutArg(inputMeshList));
if (me == 0) {
inputMeshList.print(cout,2,true,true);
cout << "\n";
}
}
else {
cout << "Cannot read input file: " << xmlMeshInFileName << "\n";
return 5;
}
// Get pamgen mesh definition
std::string meshInput = Teuchos::getParameter<std::string>(inputMeshList,
"meshInput");
/***************************************************************************/
/********************** GET CELL TOPOLOGY **********************************/
/***************************************************************************/
// Get dimensions
int dim = 3;
/***************************************************************************/
/***************************** GENERATE MESH *******************************/
/***************************************************************************/
if (me == 0) cout << "Generating mesh ... \n\n";
// Generate mesh with Pamgen
long long maxInt = 9223372036854775807LL;
Create_Pamgen_Mesh(meshInput.c_str(), dim, me, numProcs, maxInt);
// Creating mesh adapter
if (me == 0) cout << "Creating mesh adapter ... \n\n";
typedef Zoltan2::PamgenMeshAdapter<tMVector_t> inputAdapter_t;
typedef Zoltan2::EvaluatePartition<inputAdapter_t> quality_t;
typedef inputAdapter_t::part_t part_t;
typedef inputAdapter_t::base_adapter_t base_adapter_t;
inputAdapter_t *ia = new inputAdapter_t(*CommT, "region");
ia->print(me);
// Set parameters for partitioning
if (me == 0) cout << "Creating parameter list ... \n\n";
Teuchos::ParameterList params("test params");
params.set("timer_output_stream" , "std::cout");
bool do_partitioning = false;
if (action == "mj") {
do_partitioning = true;
params.set("debug_level", "basic_status");
params.set("imbalance_tolerance", 1.1);
params.set("num_global_parts", nParts);
params.set("algorithm", "multijagged");
params.set("rectilinear", "yes");
}
else if (action == "scotch") {
do_partitioning = true;
params.set("debug_level", "verbose_detailed_status");
params.set("imbalance_tolerance", 1.1);
params.set("num_global_parts", nParts);
params.set("partitioning_approach", "partition");
params.set("algorithm", "scotch");
params.set("compute_metrics","yes");
}
else if (action == "zoltan_rcb") {
do_partitioning = true;
params.set("debug_level", "verbose_detailed_status");
params.set("imbalance_tolerance", 1.1);
params.set("num_global_parts", nParts);
params.set("partitioning_approach", "partition");
params.set("algorithm", "zoltan");
}
else if (action == "zoltan_hg") {
do_partitioning = true;
params.set("debug_level", "verbose_detailed_status");
params.set("imbalance_tolerance", 1.1);
params.set("num_global_parts", nParts);
params.set("partitioning_approach", "partition");
params.set("algorithm", "zoltan");
Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
zparams.set("LB_METHOD","phg");
zparams.set("FINAL_OUTPUT", "1");
}
else if (action=="hg_ghost") {
do_partitioning = true;
params.set("debug_level", "no_status");
params.set("imbalance_tolerance", 1.1);
params.set("algorithm", "zoltan");
params.set("num_global_parts", nParts);
params.set("hypergraph_model_type","ghosting");
params.set("ghost_layers",2);
Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
zparams.set("LB_METHOD","HYPERGRAPH");
zparams.set("LB_APPROACH","PARTITION");
zparams.set("PHG_EDGE_SIZE_THRESHOLD", "1.0");
}
else if (action == "parma") {
do_partitioning = true;
params.set("debug_level", "basic_status");
params.set("imbalance_tolerance", 1.05);
params.set("algorithm", "parma");
Teuchos::ParameterList &pparams = params.sublist("parma_parameters",false);
pparams.set("parma_method","VtxElm");
}
else if (action=="zoltan_hg") {
do_partitioning = true;
params.set("debug_level", "no_status");
params.set("imbalance_tolerance", 1.1);
params.set("algorithm", "zoltan");
params.set("num_global_parts", nParts);
Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
zparams.set("LB_METHOD","HYPERGRAPH");
params.set("compute_metrics","yes");
}
else if (action == "color") {
params.set("debug_level", "verbose_detailed_status");
params.set("debug_output_file", "kdd");
params.set("debug_procs", "all");
}
if(me == 0) cout << "Action: " << action << endl;
// create Partitioning problem
if (do_partitioning) {
if (me == 0) cout << "Creating partitioning problem ... \n\n";
Zoltan2::PartitioningProblem<inputAdapter_t> problem(ia, ¶ms, CommT);
// call the partitioner
if (me == 0) cout << "Calling the partitioner ... \n\n";
problem.solve();
// An environment. This is usually created by the problem.
RCP<const Zoltan2::Environment> env = problem.getEnvironment();
RCP<const base_adapter_t> bia =
Teuchos::rcp_implicit_cast<const base_adapter_t>(rcp(ia));
// A solution (usually created by a problem)
int numLocalObj = bia->getLocalNumIDs();
int nWeights = bia->getNumWeightsPerID();
RCP<Zoltan2::PartitioningSolution<inputAdapter_t> > solution =
rcp(new Zoltan2::PartitioningSolution<inputAdapter_t>(env, CommT,
nWeights));
//Part assignment for my objects: The algorithm usually calls this.
part_t *partNum = new part_t [numLocalObj];
ArrayRCP<part_t> partAssignment(partNum, 0, numLocalObj, true);
const part_t *parts = problem.getSolution().getPartListView();
for (int i=0; i < numLocalObj; i++)
partNum[i] = parts[i];
solution->setParts(partAssignment);
RCP<const Zoltan2::PartitioningSolution<inputAdapter_t> > solutionConst =
Teuchos::rcp_const_cast<const
Zoltan2::PartitioningSolution<inputAdapter_t> >(solution);
// create metric object (also usually created by a problem)
RCP<quality_t> metricObject =
rcp(new quality_t(env, CommT, bia, solutionConst, false));
RCP<quality_t> graphMetricObject;
if (action == "scotch") {
graphMetricObject = rcp(new quality_t(env, CommT, bia, solutionConst));
}
if (!me) {
metricObject->printMetrics(cout);
problem.printMetrics(cout);
if (action == "scotch") {
graphMetricObject->printGraphMetrics(cout);
problem.printGraphMetrics(cout);
}
}
}
else {
if (me == 0) cout << "Creating coloring problem ... \n\n";
Zoltan2::ColoringProblem<inputAdapter_t> problem(ia, ¶ms);
// call the partitioner
if (me == 0) cout << "Calling the coloring algorithm ... \n\n";
problem.solve();
problem.printTimers();
}
// delete mesh
if (me == 0) cout << "Deleting the mesh ... \n\n";
Delete_Pamgen_Mesh();
if (me == 0)
std::cout << "PASS" << std::endl;
return 0;
}
int
main (int argc, char *argv[])
{
using namespace TrilinosCouplings; // Yes, this means I'm lazy.
using TpetraIntrepidPoissonExample::exactResidualNorm;
using TpetraIntrepidPoissonExample::makeMatrixAndRightHandSide;
using TpetraIntrepidPoissonExample::solveWithBelos;
using TpetraIntrepidPoissonExample::solveWithBelosGPU;
using IntrepidPoissonExample::makeMeshInput;
using IntrepidPoissonExample::parseCommandLineArguments;
using IntrepidPoissonExample::setCommandLineArgumentDefaults;
using IntrepidPoissonExample::setMaterialTensorOffDiagonalValue;
using IntrepidPoissonExample::setUpCommandLineArguments;
using Tpetra::DefaultPlatform;
using Teuchos::Comm;
using Teuchos::outArg;
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcpFromRef;
using Teuchos::getFancyOStream;
using Teuchos::FancyOStream;
using std::endl;
// Pull in typedefs from the example's namespace.
typedef TpetraIntrepidPoissonExample::ST ST;
#ifdef HAVE_TRILINOSCOUPLINGS_MUELU
typedef TpetraIntrepidPoissonExample::LO LO;
typedef TpetraIntrepidPoissonExample::GO GO;
#endif // HAVE_TRILINOSCOUPLINGS_MUELU
typedef TpetraIntrepidPoissonExample::Node Node;
typedef Teuchos::ScalarTraits<ST> STS;
typedef STS::magnitudeType MT;
typedef Teuchos::ScalarTraits<MT> STM;
typedef TpetraIntrepidPoissonExample::sparse_matrix_type sparse_matrix_type;
typedef TpetraIntrepidPoissonExample::vector_type vector_type;
typedef TpetraIntrepidPoissonExample::operator_type operator_type;
bool success = true;
try {
Teuchos::oblackholestream blackHole;
Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
const int myRank = mpiSession.getRank ();
//const int numProcs = mpiSession.getNProc ();
// Get the default communicator and Kokkos Node instance
RCP<const Comm<int> > comm =
DefaultPlatform::getDefaultPlatform ().getComm ();
RCP<Node> node = DefaultPlatform::getDefaultPlatform ().getNode ();
// Did the user specify --help at the command line to print help
// with command-line arguments?
bool printedHelp = false;
// Values of command-line arguments.
int nx, ny, nz;
std::string xmlInputParamsFile;
bool verbose, debug;
int maxNumItersFromCmdLine = -1; // -1 means "read from XML file"
double tolFromCmdLine = -1.0; // -1 means "read from XML file"
std::string solverName = "GMRES";
ST materialTensorOffDiagonalValue = 0.0;
// Set default values of command-line arguments.
setCommandLineArgumentDefaults (nx, ny, nz, xmlInputParamsFile,
solverName, verbose, debug);
// Parse and validate command-line arguments.
Teuchos::CommandLineProcessor cmdp (false, true);
setUpCommandLineArguments (cmdp, nx, ny, nz, xmlInputParamsFile,
solverName, tolFromCmdLine,
maxNumItersFromCmdLine,
verbose, debug);
cmdp.setOption ("materialTensorOffDiagonalValue",
&materialTensorOffDiagonalValue, "Off-diagonal value in "
"the material tensor. This controls the iteration count. "
"Be careful with this if you use CG, since you can easily "
"make the matrix indefinite.");
// Additional command-line arguments for GPU experimentation.
bool gpu = false;
cmdp.setOption ("gpu", "no-gpu", &gpu,
"Run example using GPU node (if supported)");
int ranks_per_node = 1;
cmdp.setOption ("ranks_per_node", &ranks_per_node,
"Number of MPI ranks per node");
int gpu_ranks_per_node = 1;
cmdp.setOption ("gpu_ranks_per_node", &gpu_ranks_per_node,
"Number of MPI ranks per node for GPUs");
int device_offset = 0;
cmdp.setOption ("device_offset", &device_offset,
"Offset for attaching MPI ranks to CUDA devices");
// Additional command-line arguments for dumping the generated
// matrix or its row Map to output files.
//
// FIXME (mfh 09 Apr 2014) Need to port these command-line
// arguments to the Epetra version.
// If matrixFilename is nonempty, dump the matrix to that file
// in MatrixMarket format.
std::string matrixFilename;
cmdp.setOption ("matrixFilename", &matrixFilename, "If nonempty, dump the "
"generated matrix to that file in MatrixMarket format.");
// If rowMapFilename is nonempty, dump the matrix's row Map to
// that file in MatrixMarket format.
std::string rowMapFilename;
cmdp.setOption ("rowMapFilename", &rowMapFilename, "If nonempty, dump the "
"generated matrix's row Map to that file in a format that "
"Tpetra::MatrixMarket::Reader can read.");
// Option to exit after building A and b (and dumping stuff to
// files, if requested).
bool exitAfterAssembly = false;
cmdp.setOption ("exitAfterAssembly", "dontExitAfterAssembly",
&exitAfterAssembly, "If true, exit after building the "
"sparse matrix and dense right-hand side vector. If either"
" --matrixFilename or --rowMapFilename are nonempty strings"
", dump the matrix resp. row Map to their respective files "
"before exiting.");
parseCommandLineArguments (cmdp, printedHelp, argc, argv, nx, ny, nz,
xmlInputParamsFile, solverName, verbose, debug);
if (printedHelp) {
// The user specified --help at the command line to print help
// with command-line arguments. We printed help already, so quit
// with a happy return code.
return EXIT_SUCCESS;
}
setMaterialTensorOffDiagonalValue (materialTensorOffDiagonalValue);
// Both streams only print on MPI Rank 0. "out" only prints if the
// user specified --verbose.
RCP<FancyOStream> out =
getFancyOStream (rcpFromRef ((myRank == 0 && verbose) ? std::cout : blackHole));
RCP<FancyOStream> err =
getFancyOStream (rcpFromRef ((myRank == 0 && debug) ? std::cerr : blackHole));
#ifdef HAVE_MPI
*out << "PARALLEL executable" << endl;
#else
*out << "SERIAL executable" << endl;
#endif
/**********************************************************************************/
/********************************** GET XML INPUTS ********************************/
/**********************************************************************************/
ParameterList inputList;
if (xmlInputParamsFile != "") {
*out << "Reading parameters from XML file \""
<< xmlInputParamsFile << "\"..." << endl;
Teuchos::updateParametersFromXmlFile (xmlInputParamsFile,
outArg (inputList));
if (myRank == 0) {
inputList.print (*out, 2, true, true);
*out << endl;
}
}
// Get Pamgen mesh definition string, either from the input
// ParameterList or from our function that makes a cube and fills in
// the number of cells along each dimension.
std::string meshInput = inputList.get("meshInput", "");
if (meshInput == "") {
*out << "Generating mesh input string: nx = " << nx
<< ", ny = " << ny
<< ", nz = " << nz << endl;
meshInput = makeMeshInput (nx, ny, nz);
}
// Total application run time
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Time", total_time);
RCP<sparse_matrix_type> A;
RCP<vector_type> B, X_exact, X;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Assembly", total_assembly);
makeMatrixAndRightHandSide (A, B, X_exact, X, comm, node, meshInput,
out, err, verbose, debug);
}
// Optionally dump the matrix and/or its row Map to files.
{
typedef Tpetra::MatrixMarket::Writer<sparse_matrix_type> writer_type;
if (matrixFilename != "") {
writer_type::writeSparseFile (matrixFilename, A);
}
if (rowMapFilename != "") {
writer_type::writeMapFile (rowMapFilename, * (A->getRowMap ()));
}
}
if (exitAfterAssembly) {
// Users might still be interested in assembly time.
Teuchos::TimeMonitor::report (comm.ptr (), std::cout);
return EXIT_SUCCESS;
}
const std::vector<MT> norms = exactResidualNorm (A, B, X_exact);
// X_exact is the exact solution of the PDE, projected onto the
// discrete mesh. It may not necessarily equal the exact solution
// of the linear system.
*out << "||B - A*X_exact||_2 = " << norms[0] << endl
<< "||B||_2 = " << norms[1] << endl
<< "||A||_F = " << norms[2] << endl;
// Setup preconditioner
std::string prec_type = inputList.get ("Preconditioner", "None");
RCP<operator_type> M;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Preconditioner Setup", total_prec);
if (prec_type == "MueLu") {
#ifdef HAVE_TRILINOSCOUPLINGS_MUELU
if (inputList.isSublist("MueLu")) {
ParameterList mueluParams = inputList.sublist("MueLu");
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A,mueluParams);
} else {
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A);
}
#else // NOT HAVE_TRILINOSCOUPLINGS_MUELU
TEUCHOS_TEST_FOR_EXCEPTION(
prec_type == "MueLu", std::runtime_error, "Tpetra scaling example: "
"In order to precondition with MueLu, you must have built Trilinos "
"with the MueLu package enabled.");
#endif // HAVE_TRILINOSCOUPLINGS_MUELU
}
} // setup preconditioner
// Get the convergence tolerance for each linear solve.
// If the user provided a nonnegative value at the command
// line, it overrides any value in the input ParameterList.
MT tol = STM::squareroot (STM::eps ()); // default value
if (tolFromCmdLine < STM::zero ()) {
tol = inputList.get ("Convergence Tolerance", tol);
} else {
tol = tolFromCmdLine;
}
// Get the maximum number of iterations for each linear solve.
// If the user provided a value other than -1 at the command
// line, it overrides any value in the input ParameterList.
int maxNumIters = 200; // default value
if (maxNumItersFromCmdLine == -1) {
maxNumIters = inputList.get ("Maximum Iterations", maxNumIters);
} else {
maxNumIters = maxNumItersFromCmdLine;
}
// Get the number of "time steps." We imitate a time-dependent
// PDE by doing this many linear solves.
const int num_steps = inputList.get ("Number of Time Steps", 1);
// Do the linear solve(s).
bool converged = false;
int numItersPerformed = 0;
if (gpu) {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total GPU Solve", total_solve);
solveWithBelosGPU (converged, numItersPerformed, tol, maxNumIters,
num_steps, ranks_per_node, gpu_ranks_per_node,
device_offset, prec_type, X, A, B, Teuchos::null, M);
}
else {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Solve", total_solve);
solveWithBelos (converged, numItersPerformed, solverName, tol,
maxNumIters, num_steps, X, A, B, Teuchos::null, M);
}
// Compute ||X-X_exact||_2
const MT norm_x = X_exact->norm2 ();
X_exact->update (-1.0, *X, 1.0);
const MT norm_error = X_exact->norm2 ();
*out << endl
<< "||X - X_exact||_2 / ||X_exact||_2 = " << norm_error / norm_x
<< endl;
} // total time block
// Summarize timings
Teuchos::TimeMonitor::report (comm.ptr (), std::cout);
} // try
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
if (success) {
return EXIT_SUCCESS;
} else {
return EXIT_FAILURE;
}
}
int main(int narg, char *arg[]) {
Teuchos::GlobalMPISession mpiSession(&narg, &arg,0);
Platform &platform = Tpetra::DefaultPlatform::getDefaultPlatform();
RCP<const Teuchos::Comm<int> > CommT = platform.getComm();
int me = CommT->getRank();
int numProcs = CommT->getSize();
/***************************************************************************/
/*************************** GET XML INPUTS ********************************/
/***************************************************************************/
// default values for command-line arguments
std::string xmlMeshInFileName("Poisson.xml");
// Read run-time options.
Teuchos::CommandLineProcessor cmdp (false, false);
cmdp.setOption("xmlfile", &xmlMeshInFileName,
"XML file with PamGen specifications");
cmdp.parse(narg, arg);
// Read xml file into parameter list
ParameterList inputMeshList;
if(xmlMeshInFileName.length()) {
if (me == 0) {
cout << "\nReading parameter list from the XML file \""
<<xmlMeshInFileName<<"\" ...\n\n";
}
Teuchos::updateParametersFromXmlFile(xmlMeshInFileName,
Teuchos::inoutArg(inputMeshList));
if (me == 0) {
inputMeshList.print(cout,2,true,true);
cout << "\n";
}
}
else {
cout << "Cannot read input file: " << xmlMeshInFileName << "\n";
return 5;
}
// Get pamgen mesh definition
std::string meshInput = Teuchos::getParameter<std::string>(inputMeshList,
"meshInput");
/***************************************************************************/
/********************** GET CELL TOPOLOGY **********************************/
/***************************************************************************/
// Get dimensions
int dim = 3;
/***************************************************************************/
/***************************** GENERATE MESH *******************************/
/***************************************************************************/
if (me == 0) cout << "Generating mesh ... \n\n";
// Generate mesh with Pamgen
long long maxInt = 9223372036854775807LL;
Create_Pamgen_Mesh(meshInput.c_str(), dim, me, numProcs, maxInt);
// Creating mesh adapter
if (me == 0) cout << "Creating mesh adapter ... \n\n";
typedef Zoltan2::PamgenMeshAdapter<tMVector_t> inputAdapter_t;
inputAdapter_t ia(*CommT, "region");
inputAdapter_t ia2(*CommT, "vertex");
inputAdapter_t::gno_t const *adjacencyIds=NULL;
inputAdapter_t::lno_t const *offsets=NULL;
ia.print(me);
Zoltan2::MeshEntityType primaryEType = ia.getPrimaryEntityType();
Zoltan2::MeshEntityType adjEType = ia.getAdjacencyEntityType();
int dimension, num_nodes, num_elem;
int error = 0;
char title[100];
int exoid = 0;
int num_elem_blk, num_node_sets, num_side_sets;
error += im_ex_get_init(exoid, title, &dimension, &num_nodes, &num_elem,
&num_elem_blk, &num_node_sets, &num_side_sets);
int *element_num_map = new int [num_elem];
error += im_ex_get_elem_num_map(exoid, element_num_map);
inputAdapter_t::gno_t *node_num_map = new int [num_nodes];
error += im_ex_get_node_num_map(exoid, node_num_map);
int *elem_blk_ids = new int [num_elem_blk];
error += im_ex_get_elem_blk_ids(exoid, elem_blk_ids);
int *num_nodes_per_elem = new int [num_elem_blk];
int *num_attr = new int [num_elem_blk];
int *num_elem_this_blk = new int [num_elem_blk];
char **elem_type = new char * [num_elem_blk];
int **connect = new int * [num_elem_blk];
for(int i = 0; i < num_elem_blk; i++){
elem_type[i] = new char [MAX_STR_LENGTH + 1];
error += im_ex_get_elem_block(exoid, elem_blk_ids[i], elem_type[i],
(int*)&(num_elem_this_blk[i]),
(int*)&(num_nodes_per_elem[i]),
(int*)&(num_attr[i]));
delete[] elem_type[i];
}
delete[] elem_type;
elem_type = NULL;
delete[] num_attr;
num_attr = NULL;
for(int b = 0; b < num_elem_blk; b++) {
connect[b] = new int [num_nodes_per_elem[b]*num_elem_this_blk[b]];
error += im_ex_get_elem_conn(exoid, elem_blk_ids[b], connect[b]);
}
delete[] elem_blk_ids;
elem_blk_ids = NULL;
int telct = 0;
if (ia.availAdjs(primaryEType, adjEType)) {
ia.getAdjsView(primaryEType, adjEType, offsets, adjacencyIds);
if ((int)ia.getLocalNumOf(primaryEType) != num_elem) {
cout << "Number of elements do not match\n";
return 2;
}
for (int b = 0; b < num_elem_blk; b++) {
for (int i = 0; i < num_elem_this_blk[b]; i++) {
if (offsets[telct + 1] - offsets[telct] != num_nodes_per_elem[b]) {
std::cout << "Number of adjacencies do not match" << std::endl;
return 3;
}
for (int j = 0; j < num_nodes_per_elem[b]; j++) {
ssize_t in_list = -1;
for(inputAdapter_t::lno_t k=offsets[telct];k<offsets[telct+1];k++) {
if(adjacencyIds[k] ==
node_num_map[connect[b][i*num_nodes_per_elem[b]+j]-1]) {
in_list = k;
break;
}
}
if (in_list < 0) {
std::cout << "Adjacency missing" << std::endl;
return 4;
}
}
++telct;
}
}
if (telct != num_elem) {
cout << "Number of elements do not match\n";
return 2;
}
}
else{
std::cout << "Adjacencies not available" << std::endl;
return 1;
}
primaryEType = ia2.getPrimaryEntityType();
adjEType = ia2.getAdjacencyEntityType();
if (ia2.availAdjs(primaryEType, adjEType)) {
ia2.getAdjsView(primaryEType, adjEType, offsets, adjacencyIds);
if ((int)ia2.getLocalNumOf(primaryEType) != num_nodes) {
cout << "Number of nodes do not match\n";
return 2;
}
telct = 0;
int *num_adj = new int[num_nodes];
for (int i = 0; i < num_nodes; i++) {
num_adj[i] = 0;
}
for (int b = 0; b < num_elem_blk; b++) {
for (int i = 0; i < num_elem_this_blk[b]; i++) {
for (int j = 0; j < num_nodes_per_elem[b]; j++) {
ssize_t in_list = -1;
++num_adj[connect[b][i * num_nodes_per_elem[b] + j] - 1];
for(inputAdapter_t::lno_t k =
offsets[connect[b][i * num_nodes_per_elem[b] + j] - 1];
k < offsets[connect[b][i * num_nodes_per_elem[b] + j]]; k++) {
if(adjacencyIds[k] == element_num_map[telct]) {
in_list = k;
break;
}
}
if (in_list < 0) {
std::cout << "Adjacency missing" << std::endl;
return 4;
}
}
++telct;
}
}
for (int i = 0; i < num_nodes; i++) {
if (offsets[i + 1] - offsets[i] != num_adj[i]) {
std::cout << "Number of adjacencies do not match" << std::endl;
return 3;
}
}
delete[] num_adj;
num_adj = NULL;
}
else{
std::cout << "Adjacencies not available" << std::endl;
return 1;
}
// delete mesh
if (me == 0) cout << "Deleting the mesh ... \n\n";
Delete_Pamgen_Mesh();
if (me == 0)
std::cout << "PASS" << std::endl;
return 0;
}
int
main (int argc, char *argv[])
{
using namespace TrilinosCouplings; // Yes, this means I'm lazy.
using TpetraIntrepidPoissonExample::exactResidualNorm;
using TpetraIntrepidPoissonExample::makeMatrixAndRightHandSide;
using TpetraIntrepidPoissonExample::solveWithBelos;
using TpetraIntrepidPoissonExample::solveWithBelosGPU;
using IntrepidPoissonExample::makeMeshInput;
using IntrepidPoissonExample::setCommandLineArgumentDefaults;
using IntrepidPoissonExample::setUpCommandLineArguments;
using IntrepidPoissonExample::parseCommandLineArguments;
using Tpetra::DefaultPlatform;
using Teuchos::Comm;
using Teuchos::outArg;
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::rcpFromRef;
using Teuchos::getFancyOStream;
using Teuchos::FancyOStream;
using std::endl;
// Pull in typedefs from the example's namespace.
typedef TpetraIntrepidPoissonExample::ST ST;
typedef TpetraIntrepidPoissonExample::LO LO;
typedef TpetraIntrepidPoissonExample::GO GO;
typedef TpetraIntrepidPoissonExample::Node Node;
typedef Teuchos::ScalarTraits<ST> STS;
typedef STS::magnitudeType MT;
typedef Teuchos::ScalarTraits<MT> STM;
typedef TpetraIntrepidPoissonExample::sparse_matrix_type sparse_matrix_type;
typedef TpetraIntrepidPoissonExample::vector_type vector_type;
typedef TpetraIntrepidPoissonExample::operator_type operator_type;
bool success = true;
try {
Teuchos::oblackholestream blackHole;
Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
const int myRank = mpiSession.getRank ();
//const int numProcs = mpiSession.getNProc ();
// Get the default communicator and Kokkos Node instance
RCP<const Comm<int> > comm =
DefaultPlatform::getDefaultPlatform ().getComm ();
RCP<Node> node = DefaultPlatform::getDefaultPlatform ().getNode ();
// Did the user specify --help at the command line to print help
// with command-line arguments?
bool printedHelp = false;
// Values of command-line arguments.
int nx, ny, nz;
std::string xmlInputParamsFile;
bool verbose, debug;
// Set default values of command-line arguments.
setCommandLineArgumentDefaults (nx, ny, nz, xmlInputParamsFile,
verbose, debug);
// Parse and validate command-line arguments.
Teuchos::CommandLineProcessor cmdp (false, true);
setUpCommandLineArguments (cmdp, nx, ny, nz, xmlInputParamsFile,
verbose, debug);
bool gpu = false;
cmdp.setOption ("gpu", "no-gpu", &gpu,
"Run example using GPU node (if supported)");
int ranks_per_node = 1;
cmdp.setOption("ranks_per_node", &ranks_per_node,
"Number of MPI ranks per node");
int gpu_ranks_per_node = 1;
cmdp.setOption("gpu_ranks_per_node", &gpu_ranks_per_node,
"Number of MPI ranks per node for GPUs");
int device_offset = 0;
cmdp.setOption("device_offset", &device_offset,
"Offset for attaching MPI ranks to CUDA devices");
parseCommandLineArguments (cmdp, printedHelp, argc, argv, nx, ny, nz,
xmlInputParamsFile, verbose, debug);
if (printedHelp) {
// The user specified --help at the command line to print help
// with command-line arguments. We printed help already, so quit
// with a happy return code.
return EXIT_SUCCESS;
}
// Both streams only print on MPI Rank 0. "out" only prints if the
// user specified --verbose.
RCP<FancyOStream> out =
getFancyOStream (rcpFromRef ((myRank == 0 && verbose) ? std::cout : blackHole));
RCP<FancyOStream> err =
getFancyOStream (rcpFromRef ((myRank == 0 && debug) ? std::cerr : blackHole));
#ifdef HAVE_MPI
*out << "PARALLEL executable" << endl;
#else
*out << "SERIAL executable" << endl;
#endif
/**********************************************************************************/
/********************************** GET XML INPUTS ********************************/
/**********************************************************************************/
ParameterList inputList;
if (xmlInputParamsFile != "") {
*out << "Reading parameters from XML file \""
<< xmlInputParamsFile << "\"..." << endl;
Teuchos::updateParametersFromXmlFile (xmlInputParamsFile,
outArg (inputList));
if (myRank == 0) {
inputList.print (*out, 2, true, true);
*out << endl;
}
}
// Get Pamgen mesh definition string, either from the input
// ParameterList or from our function that makes a cube and fills in
// the number of cells along each dimension.
std::string meshInput = inputList.get("meshInput", "");
if (meshInput == "") {
*out << "Generating mesh input string: nx = " << nx
<< ", ny = " << ny
<< ", nz = " << nz << endl;
meshInput = makeMeshInput (nx, ny, nz);
}
// Total application run time
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Time", total_time);
RCP<sparse_matrix_type> A;
RCP<vector_type> B, X_exact, X;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Assembly", total_assembly);
makeMatrixAndRightHandSide (A, B, X_exact, X, comm, node, meshInput,
out, err, verbose, debug);
}
const std::vector<MT> norms = exactResidualNorm (A, B, X_exact);
// X_exact is the exact solution of the PDE, projected onto the
// discrete mesh. It may not necessarily equal the exact solution
// of the linear system.
*out << "||B - A*X_exact||_2 = " << norms[0] << endl
<< "||B||_2 = " << norms[1] << endl
<< "||A||_F = " << norms[2] << endl;
// Setup preconditioner
std::string prec_type = inputList.get("Preconditioner", "None");
RCP<operator_type> M;
{
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Preconditioner Setup", total_prec);
if (prec_type == "MueLu") {
if (inputList.isSublist("MueLu")) {
ParameterList mueluParams = inputList.sublist("MueLu");
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A,mueluParams);
} else {
M = MueLu::CreateTpetraPreconditioner<ST,LO,GO,Node>(A);
}
}
}
bool converged = false;
int numItersPerformed = 0;
const MT tol = inputList.get("Convergence Tolerance",
STM::squareroot (STM::eps ()));
const int maxNumIters = inputList.get("Maximum Iterations", 200);
const int num_steps = inputList.get("Number of Time Steps", 1);
if (gpu) {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total GPU Solve", total_solve);
solveWithBelosGPU(converged, numItersPerformed, tol, maxNumIters, num_steps,
ranks_per_node, gpu_ranks_per_node, device_offset,
prec_type,
X, A, B, Teuchos::null, M);
}
else {
TEUCHOS_FUNC_TIME_MONITOR_DIFF("Total Solve", total_solve);
solveWithBelos (converged, numItersPerformed, tol, maxNumIters, num_steps,
X, A, B, Teuchos::null, M);
}
// Compute ||X-X_exact||_2
MT norm_x = X_exact->norm2();
X_exact->update(-1.0, *X, 1.0);
MT norm_error = X_exact->norm2();
*out << endl
<< "||X-X_exact||_2 / ||X_exact||_2 = " << norm_error / norm_x
<< endl;
} // total time block
// Summarize timings
// RCP<ParameterList> reportParams = parameterList ("TimeMonitor::report");
// reportParams->set ("Report format", std::string ("YAML"));
// reportParams->set ("writeGlobalStats", true);
// Teuchos::TimeMonitor::report (*out, reportParams);
Teuchos::TimeMonitor::summarize(std::cout);
} //try
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
if (success)
return EXIT_SUCCESS;
return EXIT_FAILURE;
}
