//---------------------------------------------------------------------------
int Solver_Belos::setup_ml_operator(AztecOO& azoo, Epetra_CrsMatrix* A)
{
#ifdef HAVE_FEI_ML
if (ml_aztec_options_ == NULL) {
ml_aztec_options_ = new int[AZ_OPTIONS_SIZE];
}
if (ml_aztec_params_ == NULL) {
ml_aztec_params_ = new double[AZ_PARAMS_SIZE];
}
if (!ml_defaults_set_) {
Teuchos::ParameterList mlparams;
ML_Epetra::SetDefaults("SA", mlparams,ml_aztec_options_,ml_aztec_params_);
mlparams.setParameters(*paramlist_);
*paramlist_ = mlparams;
ml_defaults_set_ = true;
}
if (ml_prec_ != NULL) {
delete ml_prec_; ml_prec_ = NULL;
}
ml_prec_ = new ML_Epetra::MultiLevelPreconditioner(*A, *paramlist_, true);
//azoo_->SetPrecOperator(ml_prec_);
#endif
return(0);
}
ThrustGPUNode::ThrustGPUNode (Teuchos::ParameterList &pl)
{
using std::cout;
using std::cerr;
using std::endl;
// get node parameters
Teuchos::ParameterList params = getDefaultParameters ();
params.setParameters (pl);
int device = params.get<int>("Device Number");
const int verbose = params.get<int>("Verbose");
// set device
int deviceCount; cudaGetDeviceCount(&deviceCount);
TEUCHOS_TEST_FOR_EXCEPTION(
deviceCount == 0, std::runtime_error,
"Kokkos::ThrustGPUNode constructor: system has no CUDA devices."
);
if (device < 0 || device >= deviceCount) {
cerr << "Kokkos::ThrustGPUNode constructor: The specified device number "
<< device << " is not valid. Using Device 0." << endl;
device = 0;
}
cudaDeviceProp deviceProp;
cudaSetDevice (device);
cudaGetDeviceProperties (&deviceProp, device);
// as of CUDA 2.1, device prop contains the following fields
// char name[256];
// size_t totalGlobalMem, sharedMemPerBlock;
// int regsPerBlock, warpSize;
// size_t memPitch;
// int maxThreadsPerBlock, maxThreadsDim[3], maxGridSize[3];
// size_t totalConstMem;
// int major, minor;
// int clockRate;
// size_t textureAlignment;
// int deviceOverlap;
// int multiProcessorCount;
// int kernelExecTimeoutEnabled;
if (verbose) {
cout << "Kokkos::ThrustGPUNode attached to device #" << device << " \""
<< deviceProp.name << "\", of compute capability "
<< deviceProp.major << "." << deviceProp.minor << endl;
}
totalMem_ = deviceProp.totalGlobalMem;
#if defined(HAVE_KOKKOSCLASSIC_KOKKOSCORE) && defined(KOKKOS_HAVE_CUDA)
if (! Kokkos::Cuda::host_mirror_device_type::is_initialized ()) {
Kokkos::Cuda::host_mirror_device_type::initialize ();
}
if (! Kokkos::Cuda::is_initialized ()) {
Kokkos::Cuda::initialize (Kokkos::Cuda::SelectDevice (device));
}
#endif
}
OpenMPNode::OpenMPNode (Teuchos::ParameterList &pl) :
curNumThreads_ (-1), // Default: Let OpenMP pick the number of threads
verbose_ (false) // Default: No verbose status output
{
Teuchos::ParameterList params = getDefaultParameters();
params.setParameters(pl);
const int curNumThreads = params.get<int>("Num Threads");
int verboseInt = params.get<int>("Verbose");
bool verbose = (verboseInt != 0);
if (verbose) {
std::cout << "OpenMPNode initializing with \"Num Threads\" = "
<< curNumThreads << std::endl;
}
init (curNumThreads);
curNumThreads_ = curNumThreads; // Now it's safe to set state.
verbose_ = verbose;
}
//---------------------------------------------------------------------------
int Solver_Belos::solve(fei::LinearSystem* linearSystem,
fei::Matrix* preconditioningMatrix,
const fei::ParameterSet& parameterSet,
int& iterationsTaken,
int& status)
{
std::string krylov_solver_name;
parameterSet.getStringParamValue("krylov_solver", krylov_solver_name);
Teuchos::RCP<Teuchos::ParameterList>& paramlist = paramlist_;
#ifdef HAVE_FEI_ML
if (ml_aztec_options_ == NULL)
ml_aztec_options_ = new int[AZ_OPTIONS_SIZE];
if (ml_aztec_params_ == NULL)
ml_aztec_params_ = new double[AZ_PARAMS_SIZE];
if (!ml_defaults_set_ && useML_) {
Teuchos::ParameterList mlparams;
ML_Epetra::SetDefaults("SA", mlparams, ml_aztec_options_,ml_aztec_params_);
mlparams.setParameters(*paramlist);
*paramlist = mlparams;
ml_defaults_set_ = true;
}
#endif
Trilinos_Helpers::copy_parameterset(parameterSet, *paramlist);
fei::SharedPtr<fei::Matrix> feiA = linearSystem->getMatrix();
fei::SharedPtr<fei::Vector> feix = linearSystem->getSolutionVector();
fei::SharedPtr<fei::Vector> feib = linearSystem->getRHS();
Epetra_MultiVector* x = NULL;
Epetra_MultiVector* b = NULL;
Epetra_Operator* epetra_op = 0;
Epetra_CrsMatrix* crsA = NULL;
Trilinos_Helpers::get_Epetra_pointers(feiA, feix, feib,
crsA, epetra_op, x, b);
Teuchos::RCP<Epetra_CrsMatrix> rcp_A(crsA);
Teuchos::RCP<Epetra_MultiVector> rcp_x(x);
Teuchos::RCP<Epetra_MultiVector> rcp_b(b);
if (epetra_op == 0 || x == 0 || b == 0) {
fei::console_out() << "Solver_Belos::solve Error, couldn't obtain Epetra objects"
<< " from fei container-objects."<<FEI_ENDL;
return(-1);
}
Epetra_RowMatrix* precond = NULL;
if (preconditioningMatrix != NULL) {
fei::Matrix_Impl<Epetra_CrsMatrix>* snl_epetra_crs =
dynamic_cast<fei::Matrix_Impl<Epetra_CrsMatrix>*>(preconditioningMatrix);
fei::Matrix_Impl<Epetra_VbrMatrix>* snl_epetra_vbr =
dynamic_cast<fei::Matrix_Impl<Epetra_VbrMatrix>*>(preconditioningMatrix);
if (snl_epetra_crs != NULL) {
precond = snl_epetra_crs->getMatrix().get();
}
else if (snl_epetra_vbr != NULL) {
precond = snl_epetra_vbr->getMatrix().get();
}
else {
fei::console_out() << "Solver_Belos::solve: ERROR getting epetra row matrix"
<< " from preconditioningMatrix."<<FEI_ENDL;
return(-1);
}
}
if (precond != NULL) {
//TODO: set up preconditioner for Belos here
}
bool needNewPreconditioner = false;
if (feiA->changedSinceMark()) {
feiA->markState();
needNewPreconditioner = true;
}
if (needNewPreconditioner) {
//
// if (useML_) {
#ifdef HAVE_FEI_ML
// setup_ml_operator(*azoo_, crsA);
#else
// fei::console_out() <<"Solver_Belos::solve ERROR, ML requested but HAVE_FEI_ML not defined."
// << FEI_ENDL;
// return(-1);
#endif
// }
// else {
// azoo_->SetAztecOption(AZ_pre_calc, AZ_calc);
// azoo_->SetAztecOption(AZ_keep_info, 1);
// }
}
else {
// if (!useML_) {
// azoo_->SetAztecOption(AZ_pre_calc, AZ_reuse);
// }
}
epetra_op->SetUseTranspose(useTranspose_);
Belos::SolverFactory<double,Epetra_MultiVector,Epetra_Operator> belos_factory;
belos_solver_manager_ = belos_factory.create(krylov_solver_name, paramlist);
Teuchos::RCP<Belos::LinearProblem<double,Epetra_MultiVector,Epetra_Operator> > belos_lin_prob = Teuchos::rcp(new Belos::LinearProblem<double,Epetra_MultiVector,Epetra_Operator>(rcp_A, rcp_x, rcp_b));
belos_lin_prob->setProblem();
belos_solver_manager_->setProblem(belos_lin_prob);
belos_solver_manager_->solve();
status = 0;
iterationsTaken = belos_solver_manager_->getNumIters();
rcp_A.release();
rcp_x.release();
rcp_b.release();
int olevel = 0;
parameterSet.getIntParamValue("outputLevel", olevel);
std::string param2;
parameterSet.getStringParamValue("FEI_OUTPUT_LEVEL", param2);
if (olevel >= 3 || param2 == "MATRIX_FILES" || param2 == "ALL") {
std::string param1;
parameterSet.getStringParamValue("debugOutput", param1);
FEI_OSTRINGSTREAM osstr;
if (!param1.empty()) {
osstr << param1 << "/";
}
else osstr << "./";
osstr << "x_Belos.vec";
feix->writeToFile(osstr.str().c_str());
}
return(0);
}
// =============================================================================
int
main ( int argc, char *argv[] )
{
// Initialize MPI
#ifdef HAVE_MPI
MPI_Init ( &argc,&argv );
#endif
// create Epetra communicator
#ifdef HAVE_MPI
Teuchos::RCP<Epetra_MpiComm> eComm =
Teuchos::rcp<Epetra_MpiComm> ( new Epetra_MpiComm ( MPI_COMM_WORLD ) );
#else
Teuchos::RCP<Epetra_SerialComm> eComm =
Teuchos::rcp<Epetra_SerialComm> ( new Epetra_SerialComm() );
#endif
int status;
try
{
// Create a communicator for Tpetra objects
const Teuchos::RCP<const Teuchos::Comm<int> > Comm =
Teuchos::DefaultComm<int>::getComm();
// =========================================================================
// handle command line arguments
Teuchos::CommandLineProcessor My_CLP;
My_CLP.setDocString (
"This program does continuation for the Ginzburg--Landau problem with a LOCA interface.\n"
"It is possible to give an initial guess in VTK format on the command line.\n" );
std::string xmlInputFileName = "";
My_CLP.setOption ( "xml-input-file", &xmlInputFileName,
"XML file containing the parameter list", true );
// print warning for unrecognized arguments
My_CLP.recogniseAllOptions ( true );
// don't throw exceptions
My_CLP.throwExceptions ( false );
// finally, parse the stuff!
Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn;
parseReturn = My_CLP.parse ( argc, argv );
Teuchos::RCP<Teuchos::ParameterList> paramList =
Teuchos::rcp ( new Teuchos::ParameterList );
std::cout << "Reading parameter list from \"" << xmlInputFileName << "\"."
<< std::endl;
Teuchos::updateParametersFromXmlFile ( xmlInputFileName, paramList.get() );
// =========================================================================
// extract data for the output the parameter list
Teuchos::ParameterList outputList;
outputList = paramList->sublist ( "Output", true );
// set default directory to be the directory of the XML file itself
std::string xmlPath = boost::filesystem::path ( xmlInputFileName ).branch_path().string();
boost::filesystem::path outputDirectory = outputList.get<string> ( "Output directory" );
if ( outputDirectory.root_directory().empty() ) // outputDirectory is empty or is a relative directory.
outputDirectory = xmlPath / outputDirectory;
std::string contFileBaseName =
outputList.get<string> ( "Continuation file base name" );
std::string outputFormat =
outputList.get<string> ( "Output format" );
std::string contDataFileName =
outputList.get<string> ( "Continuation data file name" );
// =========================================================================
// ---------------------------------------------------------------------------
Teuchos::ParameterList glParameters;
Teuchos::RCP<ComplexVector> psi;
Teuchos::RCP<Recti::Grid::Uniform> grid;
Teuchos::ParameterList initialGuessList;
initialGuessList = paramList->sublist ( "Initial guess", true );
boost::filesystem::path inputGuessFile = initialGuessList.get<string> ( "File name" );
if ( !inputGuessFile.empty() && inputGuessFile.root_directory().empty() ) // if inputGuessFile is a relative path
inputGuessFile = xmlPath / inputGuessFile;
TEUCHOS_ASSERT( !inputGuessFile.empty() );
// For technical reasons, the reader can only accept ComplexMultiVectors.
Teuchos::RCP<Ginla::FDM::State> state;
Recti::Grid::Reader::read ( Comm,
inputGuessFile.string(),
state,
grid,
glParameters );
// possibly overwrite the parameters
Teuchos::ParameterList & overwriteParamsList = paramList->sublist ( "Overwrite parameter list", true );
bool overwriteParameters = overwriteParamsList.get<bool> ( "Overwrite parameters" );
if ( overwriteParameters )
{
Teuchos::ParameterList & overwritePList = overwriteParamsList.sublist( "Parameters", true );
glParameters.setParameters( overwritePList );
// possibly update the scaling of the grid
grid->updateScaling( glParameters.get<double>("scaling") );
}
// ---------------------------------------------------------------------------
Teuchos::RCP<Ginla::MagneticVectorPotential::Virtual> A =
Teuchos::rcp ( new Ginla::MagneticVectorPotential::ZSquareSymmetric
( glParameters.get<double> ( "H0" ),
glParameters.get<double> ( "scaling" )
)
);
// create the operator
Teuchos::RCP<Ginla::FDM::Operator::Virtual> glOperator =
Teuchos::rcp ( new Ginla::FDM::Operator::BCCentral ( grid, A, psi->getMap(), psi->getMap() ) );
// // create a perturbation
// Teuchos::RCP<GL::Perturbation::Virtual> quadrantsPerturbation =
// Teuchos::rcp ( new GL::Perturbation::Quadrants ( grid ) );
// TODO: why not make glOperator depend upon perturbation instead?
// GinzburgLandau glProblem = GinzburgLandau ( glOperator,
// quadrantsPerturbation );
std::string fn = outputDirectory.string() + "/" + contDataFileName;
Teuchos::RCP<Ginla::IO::StatsWriter> statsWriter =
Teuchos::rcp( new Ginla::IO::StatsWriter( fn ) );
Teuchos::RCP<Ginla::FDM::LocaSystem::Bordered> glsystem;
Teuchos::ParameterList & stepperList = paramList->sublist ( "LOCA" ).sublist ( "Stepper" );
int maxLocaSteps = stepperList.get<int> ( "Max Steps" );
Teuchos::RCP<Ginla::IO::StateWriter> stateWriter =
Teuchos::rcp( new Ginla::IO::StateWriter( outputDirectory.string(),
contFileBaseName,
outputFormat,
maxLocaSteps ) );
glsystem = Teuchos::rcp ( new Ginla::FDM::LocaSystem::Bordered ( glOperator,
eComm,
psi->getMap(),
statsWriter,
stateWriter ) );
// set the initial value from glParameters
std::string contParam = stepperList.get<string> ( "Continuation Parameter" );
TEST_FOR_EXCEPTION ( !glParameters.isParameter ( contParam ),
std::logic_error,
"Parameter \"" << contParam << "\" given as continuation parameter, but doesn't exist"
<< "in the glParameters list." );
// check if the initial value was given (will be unused anyway)
if ( stepperList.isParameter ( "Initial Value" ) )
{
std::cerr << "Warning: Parameter 'LOCA->Stepper->Initial Value' given, but will not be used."
<< std::endl;
}
// TODO Get rid of the explicit "double".
stepperList.set ( "Initial Value", glParameters.get<double> ( contParam ) );
// ---------------------------------------------------------------------------
// Create the necessary objects
// ---------------------------------------------------------------------------
// Create Epetra factory
Teuchos::RCP<LOCA::Abstract::Factory> epetraFactory =
Teuchos::rcp ( new LOCA::Epetra::Factory );
// Create global data object
Teuchos::RCP<LOCA::GlobalData> globalData =
LOCA::createGlobalData ( paramList, epetraFactory );
// ---------------------------------------------------------------------------
#ifdef HAVE_LOCA_ANASAZI
Teuchos::ParameterList& eigenList =
paramList->sublist ( "LOCA" ).sublist ( "Stepper" ) .sublist ( "Eigensolver" );
std::string eigenvaluesFileName =
outputDirectory.string() + "/" + outputList.get<string> ( "Eigenvalues file name" );
std::string eigenstateFileNameAppendix =
outputList.get<string> ( "Eigenstate file name appendix" );
Teuchos::RCP<Ginla::IO::StatsWriter> eigenStatsWriter =
Teuchos::rcp( new Ginla::IO::StatsWriter( eigenvaluesFileName ) );
Teuchos::RCP<Ginla::IO::SaveEigenData> glEigenSaver =
Teuchos::RCP<Ginla::IO::SaveEigenData> ( new Ginla::IO::SaveEigenData ( eigenList,
glsystem,
stateWriter,
eigenStatsWriter ) );
Teuchos::RCP<LOCA::SaveEigenData::AbstractStrategy> glSaveEigenDataStrategy =
glEigenSaver;
eigenList.set ( "Save Eigen Data Method", "User-Defined" );
eigenList.set ( "User-Defined Save Eigen Data Name", "glSaveEigenDataStrategy" );
eigenList.set ( "glSaveEigenDataStrategy", glSaveEigenDataStrategy );
#endif
// ---------------------------------------------------------------------------
// Create all possible Epetra_Operators.
Teuchos::RCP<Epetra_RowMatrix> J = glsystem->getJacobian();
// Teuchos::RCP<Epetra_RowMatrix> M = glsystem->getPreconditioner();
// Create the linear system.
// Use the TimeDependent interface for computation of shifted matrices.
Teuchos::RCP<LOCA::Epetra::Interface::Required> iReq = glsystem;
Teuchos::RCP<NOX::Epetra::Interface::Jacobian> iJac = glsystem;
// Teuchos::RCP<NOX::Epetra::Interface::Preconditioner> iPrec = glsystem;
Teuchos::ParameterList& nlPrintParams =
paramList->sublist ( "NOX" ) .sublist ( "Printing" );
Teuchos::ParameterList& lsParams =
paramList->sublist ( "NOX" ) .sublist ( "Direction" ) .sublist ( "Newton" ) .sublist ( "Linear Solver" );
// Teuchos::RCP<NOX::Epetra::LinearSystemAztecOO> linSys = Teuchos::rcp(
// new NOX::Epetra::LinearSystemAztecOO(nlPrintParams, lsParams, iJac, J, iPrec, M, *soln));
NOX::Epetra::Vector cloneVector( Epetra_Vector( *glsystem->getMap() ) );
Teuchos::RCP<NOX::Epetra::LinearSystemAztecOO> linSys =
Teuchos::rcp ( new NOX::Epetra::LinearSystemAztecOO ( nlPrintParams,
lsParams,
iReq,
iJac,
J,
cloneVector ) );
Teuchos::RCP<LOCA::Epetra::Interface::TimeDependent> iTime = glsystem;
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
// Create a group which uses that problem interface. The group will
// be initialized to contain the default initial guess for the
// specified problem.
// translate parameters into a LOCA list
LOCA::ParameterVector locaParams =
*(Ginla::Helpers::teuchosParameterList2locaParameterVector( glParameters ));
// get initial guess
NOX::Epetra::Vector initialGuess ( glsystem->createSystemVector( *state ),
NOX::Epetra::Vector::CreateView
);
Teuchos::RCP<LOCA::Epetra::Group> grp =
Teuchos::rcp ( new LOCA::Epetra::Group ( globalData,
nlPrintParams,
iTime,
initialGuess,
linSys,
linSys,
locaParams ) );
grp->setParams ( locaParams );
// ---------------------------------------------------------------------------
// Set up the NOX status tests
Teuchos::ParameterList& noxList = paramList->sublist ( "NOX", true );
double tol = noxList.get<double> ( "Tolerance" );
int maxNonlinarSteps = noxList.get<int> ( "Max steps" );
Teuchos::RCP<NOX::StatusTest::NormF> normF =
Teuchos::rcp ( new NOX::StatusTest::NormF ( tol ) );
Teuchos::RCP<NOX::StatusTest::MaxIters> maxIters =
Teuchos::rcp ( new NOX::StatusTest::MaxIters ( maxNonlinarSteps ) );
Teuchos::RCP<NOX::StatusTest::Generic> comboOR =
Teuchos::rcp ( new NOX::StatusTest::Combo ( NOX::StatusTest::Combo::OR,
normF,
maxIters ) );
// ---------------------------------------------------------------------------
// Set up the LOCA status tests
Teuchos::RCP<LOCA::StatusTest::Abstract> maxLocaStepsTest =
Teuchos::rcp ( new LOCA::StatusTest::MaxIters ( maxLocaSteps ) );
Teuchos::RCP<LOCA::StatusTest::Abstract> zeroEnergyTest =
Teuchos::rcp ( new Ginla::StatusTest::Energy ( glsystem,
0.0 ) );
Teuchos::RCP<LOCA::StatusTest::Abstract> loopTest =
Teuchos::rcp ( new Ginla::StatusTest::Loop ( glsystem ) );
Teuchos::RCP<LOCA::StatusTest::Abstract> turnaroundTest =
Teuchos::rcp ( new Ginla::StatusTest::Turnaround () );
Teuchos::RCP<LOCA::StatusTest::Combo> locaCombo =
Teuchos::rcp ( new LOCA::StatusTest::Combo ( LOCA::StatusTest::Combo::OR ) );
locaCombo->addStatusTest( maxLocaStepsTest );
// locaCombo->addStatusTest( zeroEnergyTest );
locaCombo->addStatusTest( loopTest );
locaCombo->addStatusTest( turnaroundTest );
// ---------------------------------------------------------------------------
// Create the stepper
Teuchos::RCP<LOCA::Thyra::Group> grp2 = Teuchos::null;
Teuchos::RCP<LOCA::Stepper> stepper =
Teuchos::rcp ( new LOCA::Stepper ( globalData,
grp,
locaCombo,
comboOR,
paramList ) );
// ---------------------------------------------------------------------------
// make sure that the stepper starts off with the correct starting value
// pass pointer to stepper to glsystem to be able to read stats from the stepper in there
glsystem->setLocaStepper ( stepper );
#ifdef HAVE_LOCA_ANASAZI
glEigenSaver->setLocaStepper ( stepper );
#endif
// ---------------------------------------------------------------------------
// Perform continuation run
status = stepper->run();
// ---------------------------------------------------------------------------
// clean up
LOCA::destroyGlobalData ( globalData );
glsystem->releaseLocaStepper();
#ifdef HAVE_LOCA_ANASAZI
glEigenSaver->releaseLocaStepper();
#endif
}
catch ( std::exception &e )
{
std::cerr << e.what() << std::endl;
status += 10;
}
catch ( char const* e )
{
std::cerr << "Exception raised: " << e << std::endl;
status += 20;
}
#ifdef HAVE_MPI
MPI_Finalize();
#endif
// Final return value (0 = successful, non-zero = failure)
return status;
}
void InverseOperator::Reshape(const Operator& Op, const string Type,
Teuchos::ParameterList& List, Teuchos::ParameterList* pushlist)
{
ResetTimer();
StackPush();
Op_ = Op;
RCPRowMatrix_ = Op.GetRCPRowMatrix();
// FIXME: to add overlap and level-of-fill
int NumSweeps = List.get("smoother: sweeps", 1);
double Damping = List.get("smoother: damping factor", 0.67);
int LOF_ilu = List.get("smoother: ilu fill", 0);
double LOF_ict = List.get("smoother: ilut fill", 1.0);
double LOF_ilut = List.get("smoother: ict fill", 1.0);
string reorder = List.get("schwarz: reordering type","rcm");
Teuchos::ParameterList IFPACKList;
// any parameters from the main list List are overwritten by the pushlist
IFPACKList.set("relaxation: sweeps", NumSweeps);
IFPACKList.set("relaxation: damping factor", Damping);
IFPACKList.set("fact: level-of-fill", LOF_ilu);
IFPACKList.set("fact: ict level-of-fill", LOF_ict);
IFPACKList.set("fact: ilut level-of-fill", LOF_ilut);
IFPACKList.set("relaxation: zero starting solution", false);
IFPACKList.set("schwarz: reordering type",reorder);
IFPACKList.set("fact: relative threshold",1.0);
// if present, the pushlist is assumed to be a preconstructed ifpack list
// that is copied straight to the ifpack list here
// entries in pushlist overwrite previous list entries
if (pushlist)
IFPACKList.setParameters(*pushlist);
bool verbose = false; //(GetMyPID() == 0 && GetPrintLevel() > 5);
// the ML smoother
RCPMLPrec_ = Teuchos::null;
// The Ifpack smoother
Ifpack_Preconditioner* Prec = NULL;
if (Type == "Jacobi") {
if (verbose) {
cout << "Damping factor = " << Damping
<< ", sweeps = " << NumSweeps << endl;
cout << endl;
}
IFPACKList.set("relaxation: type", "Jacobi");
Prec = new Ifpack_PointRelaxation(RowMatrix());
}
else if (Type == "Gauss-Seidel") {
if (verbose) {
cout << "Damping factor = " << Damping
<< ", sweeps = " << NumSweeps << endl;
cout << endl;
}
IFPACKList.set("relaxation: type", "Gauss-Seidel");
Prec = new Ifpack_PointRelaxation(RowMatrix());
}
else if (Type == "symmetric Gauss-Seidel") {
if (verbose) {
cout << "Damping factor = " << Damping
<< ", sweeps = " << NumSweeps << endl;
cout << endl;
}
IFPACKList.set("relaxation: type", "symmetric Gauss-Seidel");
Prec = new Ifpack_PointRelaxation(RowMatrix());
}
else if (Type == "ILU") {
if (verbose) {
cout << "ILU factorization, ov = 0, no reordering, LOF = "
<< LOF_ilu << endl;
cout << endl;
}
// use the Additive Schwarz class because it does reordering
Prec = new Ifpack_AdditiveSchwarz<Ifpack_ILU>(RowMatrix());
}
else if (Type == "ILUT") {
if (verbose) {
cout << "ILUT factorization, ov = 0, no reordering, LOF = "
<< LOF_ilu << endl;
cout << endl;
}
Prec = new Ifpack_ILUT(RowMatrix());
}
else if (Type == "IC") {
if (verbose) {
cout << "IC factorization, ov = 0, no reordering, LOF = "
<< LOF_ilu << endl;
cout << endl;
}
Prec = new Ifpack_IC(RowMatrix());
}
else if (Type == "ICT") {
if (verbose) {
cout << "ICT factorization, ov = 0, no reordering, LOF = "
<< LOF_ilu << endl;
cout << endl;
}
Prec = new Ifpack_ICT(RowMatrix());
}
else if (Type == "LU") {
if (verbose) {
cout << "LU factorization, ov = 0, local solver = KLU" << endl;
cout << endl;
}
Prec = new Ifpack_AdditiveSchwarz<Ifpack_Amesos>(RowMatrix());
}
else if (Type == "Amesos" || Type == "Amesos-KLU") {
if (verbose) {
cout << "Amesos-KLU direct solver" << endl;
cout << endl;
}
Prec = new Ifpack_Amesos(RowMatrix());
}
else if (Type == "MLS" || Type == "ML MLS" ||
Type == "ML symmetric Gauss-Seidel" ||
Type == "ML Gauss-Seidel")
{
if (verbose) {
cout << "ML's MLS smoother" << endl;
cout << endl;
}
Teuchos::ParameterList mlparams;
ML_Epetra::SetDefaults("SA",mlparams);
int output = List.get("ML output",-47);
if (output == -47) output = List.get("output",-1);
if (output != -1) mlparams.set("ML output",output);
mlparams.set("max levels",1);
int sweeps = List.get("smoother: sweeps",1);
mlparams.set("coarse: sweeps",sweeps);
double damp = List.get("smoother: damping factor",0.67);
mlparams.set("coarse: damping factor",damp);
mlparams.set("zero starting solution", false);
if (Type == "MLS" || Type == "ML MLS")
{
mlparams.set("coarse: type","MLS"); // MLS symmetric Gauss-Seidel Amesos-KLU
int poly = List.get("smoother: MLS polynomial order",3);
mlparams.set("coarse: MLS polynomial order",poly);
}
else if (Type == "ML symmetric Gauss-Seidel")
mlparams.set("coarse: type","symmetric Gauss-Seidel"); // MLS symmetric Gauss-Seidel Amesos-KLU
else if (Type == "ML Gauss-Seidel")
mlparams.set("coarse: type","Gauss-Seidel");
else if (Type == "ML Jacobi")
mlparams.set("coarse: type","Jacobi");
else
ML_THROW("Requested type (" + Type + ") not recognized", -1);
RCPMLPrec_ = Teuchos::rcp(new ML_Epetra::MultiLevelPreconditioner(*RowMatrix(),mlparams,true));
}
else
ML_THROW("Requested type (" + Type + ") not recognized", -1);
if (Prec)
{
RCPData_ = Teuchos::rcp(Prec);
RCPData_->SetParameters(IFPACKList);
RCPData_->Initialize();
RCPData_->Compute();
UpdateFlops(RCPData_->InitializeFlops());
UpdateFlops(RCPData_->ComputeFlops());
}
else
RCPData_ = Teuchos::null;
StackPop();
UpdateTime();
}
//---------------------------------------------------------------------------
int Solver_AztecOO::solve(fei::LinearSystem* linearSystem,
fei::Matrix* preconditioningMatrix,
const fei::ParameterSet& parameterSet,
int& iterationsTaken,
int& status)
{
std::string pcstring;
parameterSet.getStringParamValue("AZ_precond", pcstring);
if (pcstring == "ML_Op") {
useML_ = true;
}
Teuchos::ParameterList& paramlist = get_ParameterList();
#ifdef HAVE_FEI_ML
if (ml_aztec_options_ == NULL)
ml_aztec_options_ = new int[AZ_OPTIONS_SIZE];
if (ml_aztec_params_ == NULL)
ml_aztec_params_ = new double[AZ_PARAMS_SIZE];
if (!ml_defaults_set_ && useML_) {
Teuchos::ParameterList mlparams;
ML_Epetra::SetDefaults("SA", mlparams, ml_aztec_options_,ml_aztec_params_);
mlparams.setParameters(paramlist);
paramlist = mlparams;
ml_defaults_set_ = true;
}
#endif
Trilinos_Helpers::copy_parameterset(parameterSet, paramlist);
fei::SharedPtr<fei::Matrix> feiA = linearSystem->getMatrix();
fei::SharedPtr<fei::Vector> feix = linearSystem->getSolutionVector();
fei::SharedPtr<fei::Vector> feib = linearSystem->getRHS();
Epetra_MultiVector* x = NULL;
Epetra_MultiVector* b = NULL;
Epetra_Operator* epetra_op = 0;
Epetra_CrsMatrix* crsA = NULL;
Trilinos_Helpers::get_Epetra_pointers(feiA, feix, feib,
crsA, epetra_op, x, b);
if (epetra_op == 0 || x == 0 || b == 0) {
fei::console_out() << "Solver_AztecOO::solve Error, couldn't obtain Epetra objects"
<< " from fei container-objects."<<FEI_ENDL;
return(-1);
}
//when we call azoo_->SetProblem, it will set some options. So we will
//first take a copy of all options and params, then reset them after the
//call to SetProblem. That way we preserve any options that have already
//been set.
std::vector<int> azoptions(AZ_OPTIONS_SIZE);
std::vector<double> azparams(AZ_PARAMS_SIZE);
const int* azoptionsptr = azoo_->GetAllAztecOptions();
const double* azparamsptr = azoo_->GetAllAztecParams();
int i;
for(i=0; i<AZ_OPTIONS_SIZE; ++i) {
azoptions[i] = azoptionsptr[i];
}
for(i=0; i<AZ_PARAMS_SIZE; ++i) {
azparams[i] = azparamsptr[i];
}
Epetra_RowMatrix* precond = NULL;
if (preconditioningMatrix != NULL) {
fei::Matrix_Impl<Epetra_CrsMatrix>* snl_epetra_crs =
dynamic_cast<fei::Matrix_Impl<Epetra_CrsMatrix>*>(preconditioningMatrix);
fei::Matrix_Impl<Epetra_VbrMatrix>* snl_epetra_vbr =
dynamic_cast<fei::Matrix_Impl<Epetra_VbrMatrix>*>(preconditioningMatrix);
if (snl_epetra_crs != NULL) {
precond = snl_epetra_crs->getMatrix().get();
}
else if (snl_epetra_vbr != NULL) {
precond = snl_epetra_vbr->getMatrix().get();
}
else {
fei::console_out() << "Solver_AztecOO::solve: ERROR getting epetra row matrix"
<< " from preconditioningMatrix."<<FEI_ENDL;
return(-1);
}
}
if (precond != NULL) {
Epetra_LinearProblem * newlinProb = new Epetra_LinearProblem(epetra_op,x,b);
azoo_->SetProblem(*newlinProb);
delete linProb;
linProb = newlinProb;
azoo_->SetAllAztecOptions(&(azoptions[0]));
azoo_->SetAllAztecParams(&(azparams[0]));
azoo_->SetUseAdaptiveDefaultsTrue();
azoo_->SetPrecMatrix(precond);
}
bool needNewPreconditioner = false;
if (feiA->changedSinceMark()) {
feiA->markState();
needNewPreconditioner = true;
}
if (needNewPreconditioner) {
Epetra_LinearProblem * newlinProb = new Epetra_LinearProblem(epetra_op,x,b);
azoo_->SetProblem(*newlinProb);
delete linProb;
linProb = newlinProb;
azoo_->SetAllAztecOptions(&(azoptions[0]));
azoo_->SetAllAztecParams(&(azparams[0]));
azoo_->SetUseAdaptiveDefaultsTrue();
if (useML_) {
#ifdef HAVE_FEI_ML
setup_ml_operator(*azoo_, crsA);
#else
fei::console_out() <<"Solver_AztecOO::solve ERROR, ML requested but HAVE_FEI_ML not defined."
<< FEI_ENDL;
return(-1);
#endif
}
else {
azoo_->SetAztecOption(AZ_pre_calc, AZ_calc);
azoo_->SetAztecOption(AZ_keep_info, 1);
}
}
else {
if (!useML_) {
azoo_->SetAztecOption(AZ_pre_calc, AZ_reuse);
}
}
epetra_op->SetUseTranspose(useTranspose_);
azoo_->SetParameters(paramlist);
maxIters_ = azoptionsptr[AZ_max_iter];
tolerance_= azparamsptr[AZ_tol];
status = azoo_->Iterate(maxIters_,
//2,//maxSolveAttempts
tolerance_);
iterationsTaken = azoo_->NumIters();
int olevel = 0;
parameterSet.getIntParamValue("outputLevel", olevel);
std::string param2;
parameterSet.getStringParamValue("FEI_OUTPUT_LEVEL", param2);
if (olevel >= 3 || param2 == "MATRIX_FILES" || param2 == "ALL") {
std::string param1;
parameterSet.getStringParamValue("debugOutput", param1);
FEI_OSTRINGSTREAM osstr;
if (!param1.empty()) {
osstr << param1 << "/";
}
else osstr << "./";
osstr << "x_AztecOO.vec";
feix->writeToFile(osstr.str().c_str());
}
return(0);
}