Piro::SteadyStateSolver<Scalar>::
SteadyStateSolver(
const Teuchos::RCP<const Thyra::ModelEvaluator<Scalar> > &model,
int numParameters) :
model_(model),
num_p_(numParameters),
num_g_(model->Ng())
{}
void Piro::RythmosSolver<Scalar>::initialize(
#endif
const Teuchos::RCP<Teuchos::ParameterList> &appParams,
const Teuchos::RCP< Thyra::ModelEvaluator<Scalar> > &in_model,
const Teuchos::RCP<Rythmos::IntegrationObserverBase<Scalar> > &observer)
{
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::RCP;
using Teuchos::rcp;
// set some internals
model = in_model;
num_p = in_model->Np();
num_g = in_model->Ng();
//
*out << "\nA) Get the base parameter list ...\n";
//
if (appParams->isSublist("Rythmos")) {
RCP<Teuchos::ParameterList> rythmosPL = sublist(appParams, "Rythmos", true);
rythmosPL->validateParameters(*getValidRythmosParameters(),0);
{
const std::string verbosity = rythmosPL->get("Verbosity Level", "VERB_DEFAULT");
if (verbosity == "VERB_NONE") solnVerbLevel = Teuchos::VERB_NONE;
else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT;
else if (verbosity == "VERB_LOW") solnVerbLevel = Teuchos::VERB_LOW;
else if (verbosity == "VERB_MEDIUM") solnVerbLevel = Teuchos::VERB_MEDIUM;
else if (verbosity == "VERB_HIGH") solnVerbLevel = Teuchos::VERB_HIGH;
else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME;
else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Unknown verbosity option specified in Piro_RythmosSolver.");
}
t_initial = rythmosPL->get("Initial Time", 0.0);
t_final = rythmosPL->get("Final Time", 0.1);
const std::string stepperType = rythmosPL->get("Stepper Type", "Backward Euler");
//
*out << "\nC) Create and initalize the forward model ...\n";
//
*out << "\nD) Create the stepper and integrator for the forward problem ...\n";
//
if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "Rythmos") {
Teuchos::RCP<Rythmos::TimeStepNonlinearSolver<Scalar> > rythmosTimeStepSolver =
Rythmos::timeStepNonlinearSolver<Scalar>();
if (rythmosPL->getEntryPtr("NonLinear Solver")) {
RCP<Teuchos::ParameterList> nonlinePL =
sublist(rythmosPL, "NonLinear Solver", true);
rythmosTimeStepSolver->setParameterList(nonlinePL);
}
fwdTimeStepSolver = rythmosTimeStepSolver;
}
else if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "NOX") {
#ifdef HAVE_PIRO_NOX
Teuchos::RCP<Thyra::NOXNonlinearSolver> nox_solver = Teuchos::rcp(new Thyra::NOXNonlinearSolver);
Teuchos::RCP<Teuchos::ParameterList> nox_params = Teuchos::rcp(new Teuchos::ParameterList);
*nox_params = appParams->sublist("NOX");
nox_solver->setParameterList(nox_params);
fwdTimeStepSolver = nox_solver;
#else
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Requested NOX solver for a Rythmos Transient solve, Trilinos was not built with NOX enabled. Please rebuild Trilinos or use the native Rythmos nonlinear solver.");
#endif
}
if (stepperType == "Backward Euler") {
fwdStateStepper = Rythmos::backwardEulerStepper<Scalar> (model, fwdTimeStepSolver);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "Forward Euler") {
fwdStateStepper = Rythmos::forwardEulerStepper<Scalar> (model);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "Explicit RK") {
fwdStateStepper = Rythmos::explicitRKStepper<Scalar>(model);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "BDF") {
Teuchos::RCP<Teuchos::ParameterList> BDFparams =
Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);
Teuchos::RCP<Teuchos::ParameterList> BDFStepControlPL =
Teuchos::sublist(BDFparams,"Step Control Settings");
fwdStateStepper = Teuchos::rcp( new Rythmos::ImplicitBDFStepper<Scalar>(model,fwdTimeStepSolver,BDFparams) );
fwdStateStepper->setInitialCondition(model->getNominalValues());
}
else {
// first (before failing) check to see if the user has added stepper factory
typename std::map<std::string,Teuchos::RCP<Piro::RythmosStepperFactory<Scalar> > >::const_iterator
stepFactItr = stepperFactories.find(stepperType);
if(stepFactItr!=stepperFactories.end()) {
// the user has added it, hot dog lets build a new stepper!
Teuchos::RCP<Teuchos::ParameterList> stepperParams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);
// build the stepper using the factory
fwdStateStepper = stepFactItr->second->buildStepper(model,fwdTimeStepSolver,stepperParams);
// the user decided to override the model being used (let them)
if(fwdStateStepper->getModel()!=model && fwdStateStepper->getModel()!=Teuchos::null) {
model = Teuchos::rcp_const_cast<Thyra::ModelEvaluator<Scalar> >(fwdStateStepper->getModel());
num_p = in_model->Np();
num_g = in_model->Ng();
}
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(
true, Teuchos::Exceptions::InvalidParameter,
std::endl << "Error! Piro::RythmosSolver: Invalid Steper Type: "
<< stepperType << std::endl);
}
}
// Step control strategy
{
// If the stepper can accept a step control strategy, then attempt to build one.
RCP<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> > scsa_stepper =
Teuchos::rcp_dynamic_cast<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> >(fwdStateStepper);
if (Teuchos::nonnull(scsa_stepper)) {
const std::string step_control_strategy = rythmosPL->get("Step Control Strategy Type", "None");
if (step_control_strategy == "None") {
// don't do anything, stepper will build default
} else if (step_control_strategy == "ImplicitBDFRamping") {
const RCP<Rythmos::ImplicitBDFStepperRampingStepControl<Scalar> > rscs =
rcp(new Rythmos::ImplicitBDFStepperRampingStepControl<Scalar>);
const RCP<ParameterList> p = parameterList(rythmosPL->sublist("Rythmos Step Control Strategy"));
rscs->setParameterList(p);
scsa_stepper->setStepControlStrategy(rscs);
}
else {
// first (before failing) check to see if the user has added step control factory
typename std::map<std::string,Teuchos::RCP<Piro::RythmosStepControlFactory<Scalar> > >::const_iterator
stepControlFactItr = stepControlFactories.find(step_control_strategy);
if (stepControlFactItr != stepControlFactories.end())
{
const RCP<Rythmos::StepControlStrategyBase<Scalar> > rscs = stepControlFactItr->second->buildStepControl();
const RCP<ParameterList> p = parameterList(rythmosPL -> sublist("Rythmos Step Control Strategy"));
rscs->setParameterList(p);
scsa_stepper->setStepControlStrategy(rscs);
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
"Error! Piro::RythmosSolver: Invalid step control strategy type: "
<< step_control_strategy << std::endl);
}
}
}
}
{
const RCP<Teuchos::ParameterList> integrationControlPL =
Teuchos::sublist(rythmosPL, "Rythmos Integration Control", true);
RCP<Rythmos::DefaultIntegrator<Scalar> > defaultIntegrator;
if (rythmosPL->get("Rythmos Integration Control Strategy", "Simple") == "Simple") {
defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL));
}
else if(rythmosPL->get<std::string>("Rythmos Integration Control Strategy") == "Ramping") {
defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::rampingIntegrationControlStrategy<Scalar>(integrationControlPL));
}
fwdStateIntegrator = defaultIntegrator;
}
fwdStateIntegrator->setParameterList(sublist(rythmosPL, "Rythmos Integrator", true));
if (Teuchos::nonnull(observer)) {
fwdStateIntegrator->setIntegrationObserver(observer);
}
}
else if (appParams->isSublist("Rythmos Solver")) {
/** New parameter list format **/
RCP<Teuchos::ParameterList> rythmosSolverPL = sublist(appParams, "Rythmos Solver", true);
RCP<Teuchos::ParameterList> rythmosPL = sublist(rythmosSolverPL, "Rythmos", true);
{
const std::string verbosity = rythmosSolverPL->get("Verbosity Level", "VERB_DEFAULT");
if (verbosity == "VERB_NONE") solnVerbLevel = Teuchos::VERB_NONE;
else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT;
else if (verbosity == "VERB_LOW") solnVerbLevel = Teuchos::VERB_LOW;
else if (verbosity == "VERB_MEDIUM") solnVerbLevel = Teuchos::VERB_MEDIUM;
else if (verbosity == "VERB_HIGH") solnVerbLevel = Teuchos::VERB_HIGH;
else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME;
else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
"Unknown verbosity option specified in Piro_RythmosSolver.");
}
t_initial = rythmosPL->sublist("Integrator Settings").get("Initial Time", 0.0);
t_final = rythmosPL->sublist("Integrator Settings").get("Final Time", 0.1);
const std::string stepperType = rythmosPL->sublist("Stepper Settings")
.sublist("Stepper Selection").get("Stepper Type", "Backward Euler");
//
// *out << "\nB) Create the Stratimikos linear solver factory ...\n";
//
// This is the linear solve strategy that will be used to solve for the
// linear system with the W.
//
Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder;
#ifdef HAVE_PIRO_IFPACK2
typedef Thyra::PreconditionerFactoryBase<double> Base;
#ifdef ALBANY_BUILD
typedef Thyra::Ifpack2PreconditionerFactory<Tpetra::CrsMatrix<double, LocalOrdinal, GlobalOrdinal, Node> > Impl;
#else
typedef Thyra::Ifpack2PreconditionerFactory<Tpetra::CrsMatrix<double> > Impl;
#endif
linearSolverBuilder.setPreconditioningStrategyFactory(Teuchos::abstractFactoryStd<Base, Impl>(), "Ifpack2");
#endif
#ifdef HAVE_PIRO_MUELU
#ifdef ALBANY_BUILD
Stratimikos::enableMueLu<LocalOrdinal, GlobalOrdinal, Node>(linearSolverBuilder);
#else
Stratimikos::enableMueLu(linearSolverBuilder);
#endif
#endif
linearSolverBuilder.setParameterList(sublist(rythmosSolverPL, "Stratimikos", true));
rythmosSolverPL->validateParameters(*getValidRythmosSolverParameters(),0);
RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory =
createLinearSolveStrategy(linearSolverBuilder);
//
*out << "\nC) Create and initalize the forward model ...\n";
//
// C.1) Create the underlying EpetraExt::ModelEvaluator
// already constructed as "model". Decorate if needed.
// TODO: Generelize to any explicit method, option to invert mass matrix
if (stepperType == "Explicit RK") {
if (rythmosSolverPL->get("Invert Mass Matrix", false)) {
Teuchos::RCP<Thyra::ModelEvaluator<Scalar> > origModel = model;
rythmosSolverPL->get("Lump Mass Matrix", false); //JF line does not do anything
model = Teuchos::rcp(new Piro::InvertMassMatrixDecorator<Scalar>(
sublist(rythmosSolverPL,"Stratimikos", true), origModel,
true,rythmosSolverPL->get("Lump Mass Matrix", false),false));
}
}
// C.2) Create the Thyra-wrapped ModelEvaluator
thyraModel = rcp(new Thyra::DefaultModelEvaluatorWithSolveFactory<Scalar>(model, lowsFactory));
const RCP<const Thyra::VectorSpaceBase<double> > x_space =
thyraModel->get_x_space();
//
*out << "\nD) Create the stepper and integrator for the forward problem ...\n";
//
fwdTimeStepSolver = Rythmos::timeStepNonlinearSolver<double>();
if (rythmosSolverPL->getEntryPtr("NonLinear Solver")) {
const RCP<Teuchos::ParameterList> nonlinePL =
sublist(rythmosSolverPL, "NonLinear Solver", true);
fwdTimeStepSolver->setParameterList(nonlinePL);
}
// Force Default Integrator since this is needed for Observers
rythmosPL->sublist("Integrator Settings").sublist("Integrator Selection").
set("Integrator Type","Default Integrator");
RCP<Rythmos::IntegratorBuilder<double> > ib = Rythmos::integratorBuilder<double>();
ib->setParameterList(rythmosPL);
Thyra::ModelEvaluatorBase::InArgs<double> ic = thyraModel->getNominalValues();
RCP<Rythmos::IntegratorBase<double> > integrator = ib->create(thyraModel,ic,fwdTimeStepSolver);
fwdStateIntegrator = Teuchos::rcp_dynamic_cast<Rythmos::DefaultIntegrator<double> >(integrator,true);
fwdStateStepper = fwdStateIntegrator->getNonconstStepper();
if (Teuchos::nonnull(observer))
fwdStateIntegrator->setIntegrationObserver(observer);
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(
appParams->isSublist("Rythmos") || appParams->isSublist("Rythmos Solver"),
Teuchos::Exceptions::InvalidParameter, std::endl <<
"Error! Piro::RythmosSolver: must have either Rythmos or Rythmos Solver sublist ");
}
isInitialized = true;
}
// The solve is done in the felix_driver_run function, and the solution is passed back to Glimmer-CISM
// IK, 12/3/13: time_inc_yr and cur_time_yr are not used here...
void felix_driver_run(FelixToGlimmer * ftg_ptr, double& cur_time_yr, double time_inc_yr)
{
//IK, 12/9/13: how come FancyOStream prints an all processors??
Teuchos::RCP<Teuchos::FancyOStream> out(Teuchos::VerboseObjectBase::getDefaultOStream());
if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0) {
std::cout << "In felix_driver_run, cur_time, time_inc = " << cur_time_yr
<< " " << time_inc_yr << std::endl;
}
// ---------------------------------------------
// get u and v velocity solution from Glimmer-CISM
// IK, 11/26/13: need to concatenate these into a single solve for initial condition for Albany/FELIX solve
// IK, 3/14/14: moved this step to felix_driver_run from felix_driver init, since we still want to grab and u and v velocities for CISM if the mesh hasn't changed,
// in which case only felix_driver_run will be called, not felix_driver_init.
// ---------------------------------------------
if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0)
std::cout << "In felix_driver_run: grabbing pointers to u and v velocities in CISM..." << std::endl;
uVel_ptr = ftg_ptr ->getDoubleVar("uvel", "velocity");
vVel_ptr = ftg_ptr ->getDoubleVar("vvel", "velocity");
// ---------------------------------------------
// Set restart solution to the one passed from CISM
// IK, 3/14/14: moved this from felix_driver_init to felix_driver_run.
// ---------------------------------------------
if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0)
std::cout << "In felix_driver_run: setting initial condition from CISM..." << std::endl;
//Check what kind of ordering you have in the solution & create solutionField object.
interleavedOrdering = meshStruct->getInterleavedOrdering();
Albany::AbstractSTKFieldContainer::VectorFieldType* solutionField;
if(interleavedOrdering)
solutionField = Teuchos::rcp_dynamic_cast<Albany::OrdinarySTKFieldContainer<true> >(meshStruct->getFieldContainer())->getSolutionField();
else
solutionField = Teuchos::rcp_dynamic_cast<Albany::OrdinarySTKFieldContainer<false> >(meshStruct->getFieldContainer())->getSolutionField();
//Create vector used to renumber nodes on each processor from the Albany convention (horizontal levels first) to the CISM convention (vertical layers first)
nNodes2D = (global_ewn + 1)*(global_nsn+1); //number global nodes in the domain in 2D
nNodesProc2D = (nsn-2*nhalo+1)*(ewn-2*nhalo+1); //number of nodes on each processor in 2D
cismToAlbanyNodeNumberMap.resize(upn*nNodesProc2D);
for (int j=0; j<nsn-2*nhalo+1;j++) {
for (int i=0; i<ewn-2*nhalo+1; i++) {
for (int k=0; k<upn; k++) {
int index = k+upn*i + j*(ewn-2*nhalo+1)*upn;
cismToAlbanyNodeNumberMap[index] = k*nNodes2D + global_node_id_owned_map_Ptr[i+j*(ewn-2*nhalo+1)];
//if (mpiComm->MyPID() == 0)
// std::cout << "index: " << index << ", cismToAlbanyNodeNumberMap: " << cismToAlbanyNodeNumberMap[index] << std::endl;
}
}
}
//The way it worked out, uVel_ptr and vVel_ptr have more nodes than the nodes in the mesh passed to Albany/CISM for the solve. In particular,
//there is 1 row of halo elements in uVel_ptr and vVel_ptr. To account for this, we copy uVel_ptr and vVel_ptr into std::vectors, which do not have the halo elements.
std::vector<double> uvel_vec(upn*nNodesProc2D);
std::vector<double> vvel_vec(upn*nNodesProc2D);
int counter1 = 0;
int counter2 = 0;
int local_nodeID;
for (int j=0; j<nsn-1; j++) {
for (int i=0; i<ewn-1; i++) {
for (int k=0; k<upn; k++) {
if (j >= nhalo-1 & j < nsn-nhalo) {
if (i >= nhalo-1 & i < ewn-nhalo) {
#ifdef CISM_USE_EPETRA
local_nodeID = node_map->LID(cismToAlbanyNodeNumberMap[counter1]);
#else
local_nodeID = node_map->getLocalElement(cismToAlbanyNodeNumberMap[counter1]);
#endif
uvel_vec[counter1] = uVel_ptr[counter2];
vvel_vec[counter1] = vVel_ptr[counter2];
counter1++;
}
}
counter2++;
}
}
}
//Loop over all the elements to find which nodes are active. For the active nodes, copy uvel and vvel from CISM into Albany solution array to
//use as initial condition.
//NOTE: there is some inefficiency here by looping over all the elements. TO DO? pass only active nodes from Albany-CISM to improve this?
double velScale = seconds_per_year*vel_scaling_param;
for (int i=0; i<nElementsActive; i++) {
for (int j=0; j<8; j++) {
int node_GID = global_element_conn_active_Ptr[i + nElementsActive*j]; //node_GID is 1-based
#ifdef CISM_USE_EPETRA
int node_LID = node_map->LID(node_GID); //node_LID is 0-based
#else
int node_LID = node_map->getLocalElement(node_GID); //node_LID is 0-based
#endif
stk::mesh::Entity node = meshStruct->bulkData->get_entity(stk::topology::NODE_RANK, node_GID);
double* sol = stk::mesh::field_data(*solutionField, node);
//IK, 3/18/14: added division by velScale to convert uvel and vvel from dimensionless to having units of m/year (the Albany units)
sol[0] = uvel_vec[node_LID]/velScale;
sol[1] = vvel_vec[node_LID]/velScale;
}
}
// ---------------------------------------------------------------------------------------------------
// Solve
// ---------------------------------------------------------------------------------------------------
if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0)
std::cout << "In felix_driver_run: starting the solve... " << std::endl;
//Need to set HasRestart solution such that uvel_Ptr and vvel_Ptr (u and v from Glimmer/CISM) are always set as initial condition?
meshStruct->setHasRestartSolution(!first_time_step);
//Turn off homotopy if we're not in the first time-step.
//NOTE - IMPORTANT: Glen's Law Homotopy parameter should be set to 1.0 in the parameter list for this logic to work!!!
if (!first_time_step)
{
meshStruct->setRestartDataTime(parameterList->sublist("Problem").get("Homotopy Restart Step", 1.));
double homotopy = parameterList->sublist("Problem").sublist("FELIX Viscosity").get("Glen's Law Homotopy Parameter", 1.0);
if(meshStruct->restartDataTime()== homotopy) {
parameterList->sublist("Problem").set("Solution Method", "Steady");
parameterList->sublist("Piro").set("Solver Type", "NOX");
}
}
albanyApp->createDiscretization();
//IK, 10/30/14: Check that # of elements from previous time step hasn't changed.
//If it has not, use previous solution as initial guess for current time step.
//Otherwise do not set initial solution. It's possible this can be improved so some part of the previous solution is used
//defined on the current mesh (if it receded, which likely it will in dynamic ice sheet simulations...).
if (nElementsActivePrevious != nElementsActive) previousSolution = Teuchos::null;
albanyApp->finalSetUp(parameterList, previousSolution);
//if (!first_time_step)
// std::cout << "previousSolution: " << *previousSolution << std::endl;
#ifdef CISM_USE_EPETRA
solver = slvrfctry->createThyraSolverAndGetAlbanyApp(albanyApp, mpiComm, mpiComm, Teuchos::null, false);
#else
solver = slvrfctry->createAndGetAlbanyAppT(albanyApp, mpiCommT, mpiCommT, Teuchos::null, false);
#endif
Teuchos::ParameterList solveParams;
solveParams.set("Compute Sensitivities", true);
Teuchos::Array<Teuchos::RCP<const Thyra::VectorBase<double> > > thyraResponses;
Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Thyra::MultiVectorBase<double> > > > thyraSensitivities;
Piro::PerformSolveBase(*solver, solveParams, thyraResponses, thyraSensitivities);
#ifdef CISM_USE_EPETRA
const Epetra_Map& ownedMap(*albanyApp->getDiscretization()->getMap()); //owned map
const Epetra_Map& overlapMap(*albanyApp->getDiscretization()->getOverlapMap()); //overlap map
Epetra_Import import(overlapMap, ownedMap); //importer from ownedMap to overlapMap
Epetra_Vector solutionOverlap(overlapMap); //overlapped solution
solutionOverlap.Import(*albanyApp->getDiscretization()->getSolutionField(), import, Insert);
#else
Teuchos::RCP<const Tpetra_Map> ownedMap = albanyApp->getDiscretization()->getMapT(); //owned map
Teuchos::RCP<const Tpetra_Map> overlapMap = albanyApp->getDiscretization()->getOverlapMapT(); //overlap map
Teuchos::RCP<Tpetra_Import> import = Teuchos::rcp(new Tpetra_Import(ownedMap, overlapMap));
Teuchos::RCP<Tpetra_Vector> solutionOverlap = Teuchos::rcp(new Tpetra_Vector(overlapMap));
solutionOverlap->doImport(*albanyApp->getDiscretization()->getSolutionFieldT(), *import, Tpetra::INSERT);
Teuchos::ArrayRCP<const ST> solutionOverlap_constView = solutionOverlap->get1dView();
#endif
#ifdef WRITE_TO_MATRIX_MARKET
#ifdef CISM_USE_EPETRA
//For debug: write solution and maps to matrix market file
EpetraExt::BlockMapToMatrixMarketFile("node_map.mm", *node_map);
EpetraExt::BlockMapToMatrixMarketFile("map.mm", ownedMap);
EpetraExt::BlockMapToMatrixMarketFile("overlap_map.mm", overlapMap);
EpetraExt::MultiVectorToMatrixMarketFile("solution.mm", *albanyApp->getDiscretization()->getSolutionField());
#else
Tpetra_MatrixMarket_Writer::writeMapFile("node_map.mm", *node_map);
Tpetra_MatrixMarket_Writer::writeMapFile("map.mm", *ownedMap);
Tpetra_MatrixMarket_Writer::writeMapFile("overlap_map.mm", *overlapMap);
Tpetra_MatrixMarket_Writer::writeDenseFile("solution.mm", app->getDiscretization()->getSolutionFieldT());
#endif
#endif
//set previousSolution (used as initial guess for next time step) to final Albany solution.
previousSolution = Teuchos::rcp(new Tpetra_Vector(*albanyApp->getDiscretization()->getSolutionFieldT()));
nElementsActivePrevious = nElementsActive;
//std::cout << "Final solution: " << *albanyApp->getDiscretization()->getSolutionField() << std::endl;
// ---------------------------------------------------------------------------------------------------
// Compute sensitivies / responses and perform regression tests
// IK, 12/9/13: how come this is turned off in mpas branch?
// ---------------------------------------------------------------------------------------------------
if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0)
std::cout << "Computing responses and sensitivities..." << std::endl;
int status=0; // 0 = pass, failures are incremented
#ifdef CISM_USE_EPETRA
Teuchos::Array<Teuchos::RCP<const Epetra_Vector> > responses;
Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Epetra_MultiVector> > > sensitivities;
epetraFromThyra(mpiComm, thyraResponses, thyraSensitivities, responses, sensitivities);
#else
Teuchos::Array<Teuchos::RCP<const Tpetra_Vector> > responses;
Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Tpetra_MultiVector> > > sensitivities;
tpetraFromThyra(thyraResponses, thyraSensitivities, responses, sensitivities);
#endif
const int num_p = solver->Np(); // Number of *vectors* of parameters
const int num_g = solver->Ng(); // Number of *vectors* of responses
if (debug_output_verbosity != 0) {
*out << "Finished eval of first model: Params, Responses "
<< std::setprecision(12) << std::endl;
}
const Thyra::ModelEvaluatorBase::InArgs<double> nominal = solver->getNominalValues();
if (debug_output_verbosity != 0) {
for (int i=0; i<num_p; i++) {
#ifdef CISM_USE_EPETRA
const Teuchos::RCP<const Epetra_Vector> p_init = epetraVectorFromThyra(mpiComm, nominal.get_p(i));
p_init->Print(*out << "\nParameter vector " << i << ":\n");
#else
Albany::printTpetraVector(*out << "\nParameter vector " << i << ":\n",
ConverterT::getConstTpetraVector(nominal.get_p(i)));
#endif
}
}
for (int i=0; i<num_g-1; i++) {
#ifdef CISM_USE_EPETRA
const Teuchos::RCP<const Epetra_Vector> g = responses[i];
#else
const Teuchos::RCP<const Tpetra_Vector> g = responses[i];
#endif
bool is_scalar = true;
if (albanyApp != Teuchos::null)
is_scalar = albanyApp->getResponse(i)->isScalarResponse();
if (is_scalar) {
if (debug_output_verbosity != 0) {
#ifdef CISM_USE_EPETRA
g->Print(*out << "\nResponse vector " << i << ":\n");
#else
Albany::printTpetraVector(*out << "\nResponse vector " << i << ":\n", g);
#endif
}
if (num_p == 0 && cur_time_yr == final_time) {
// Just calculate regression data -- only if in final time step
#ifdef CISM_USE_EPETRA
status += slvrfctry->checkSolveTestResults(i, 0, g.get(), NULL);
#else
status += slvrfctry->checkSolveTestResultsT(i, 0, g.get(), NULL);
#endif
} else {
for (int j=0; j<num_p; j++) {
#ifdef CISM_USE_EPETRA
const Teuchos::RCP<const Epetra_MultiVector> dgdp = sensitivities[i][j];
#else
const Teuchos::RCP<const Tpetra_MultiVector> dgdp = sensitivities[i][j];
#endif
if (debug_output_verbosity != 0) {
if (Teuchos::nonnull(dgdp)) {
#ifdef CISM_USE_EPETRA
dgdp->Print(*out << "\nSensitivities (" << i << "," << j << "):!\n");
#else
Albany::printTpetraVector(*out << "\nSensitivities (" << i << "," << j << "):!\n", dgdp);
#endif
}
}
if (cur_time_yr == final_time) {
#ifdef CISM_USE_EPETRA
status += slvrfctry->checkSolveTestResults(i, j, g.get(), dgdp.get());
#else
status += slvrfctry->checkSolveTestResultsT(i, j, g.get(), dgdp.get());
#endif
}
}
}
}
}
if (debug_output_verbosity != 0 && cur_time_yr == final_time) //only print regression test result if you're in the final time step
*out << "\nNumber of Failed Comparisons: " << status << std::endl;
//IK, 10/30/14: added the following line so that when you run ctest from CISM the test fails if there are some failed comparisons.
if (status > 0)
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, "All regression comparisons did not pass!" << std::endl);
// ---------------------------------------------------------------------------------------------------
// Copy solution back to glimmer uvel and vvel arrays to be passed back
// ---------------------------------------------------------------------------------------------------
//std::cout << "overlapMap # global elements: " << overlapMap.NumGlobalElements() << std::endl;
//std::cout << "overlapMap # my elements: " << overlapMap.NumMyElements() << std::endl;
//std::cout << "overlapMap: " << overlapMap << std::endl;
//std::cout << "map # global elements: " << ownedMap.NumGlobalElements() << std::endl;
//std::cout << "map # my elements: " << ownedMap.NumMyElements() << std::endl;
//std::cout << "node_map # global elements: " << node_map->NumGlobalElements() << std::endl;
//std::cout << "node_map # my elements: " << node_map->NumMyElements() << std::endl;
//std::cout << "node_map: " << *node_map << std::endl;
if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0)
std::cout << "In felix_driver_run: copying Albany solution to uvel and vvel to send back to CISM... " << std::endl;
#ifdef CISM_USE_EPETRA
//Epetra_Vectors to hold uvel and vvel to be passed to Glimmer/CISM
Epetra_Vector uvel(*node_map, true);
Epetra_Vector vvel(*node_map, true);
#else
//Tpetra_Vectors to hold uvel and vvel to be passed to Glimmer/CISM
Teuchos::RCP<Tpetra_Vector> uvel = Teuchos::rcp(new Tpetra_Vector(node_map, true));
Teuchos::RCP<Tpetra_Vector> vvel = Teuchos::rcp(new Tpetra_Vector(node_map, true));
#endif
#ifdef CISM_USE_EPETRA
if (interleavedOrdering == true) {
for (int i=0; i<overlapMap.NumMyElements(); i++) {
int global_dof = overlapMap.GID(i);
double sol_value = solutionOverlap[i];
int modulo = (global_dof % 2); //check if dof is for u or for v
int vel_global_dof, vel_local_dof;
if (modulo == 0) { //u dof
vel_global_dof = global_dof/2+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map
//std::cout << "uvel: global_dof = " << global_dof << ", uvel_global_dof = " << vel_global_dof << ", uvel_local_dof = " << vel_local_dof << std::endl;
uvel.ReplaceMyValues(1, &sol_value, &vel_local_dof);
}
else { // v dof
vel_global_dof = (global_dof-1)/2+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map
vvel.ReplaceMyValues(1, &sol_value, & vel_local_dof);
}
}
}
else { //note: the case with non-interleaved ordering has not been tested...
int numDofs = overlapMap.NumGlobalElements();
for (int i=0; i<overlapMap.NumMyElements(); i++) {
int global_dof = overlapMap.GID(i);
double sol_value = solutionOverlap[i];
int vel_global_dof, vel_local_dof;
if (global_dof < numDofs/2) { //u dof
vel_global_dof = global_dof+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map
uvel.ReplaceMyValues(1, &sol_value, &vel_local_dof);
}
else { //v dofs
vel_global_dof = global_dof-numDofs/2+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map
vvel.ReplaceMyValues(1, &sol_value, & vel_local_dof);
}
}
}
#else
if (interleavedOrdering == true) {
for (int i=0; i<overlapMap->getNodeNumElements(); i++) {
int global_dof = overlapMap->getGlobalElement(i);
double sol_value = solutionOverlap_constView[i];
int modulo = (global_dof % 2); //check if dof is for u or for v
int vel_global_dof, vel_local_dof;
if (modulo == 0) { //u dof
vel_global_dof = global_dof/2+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map
//std::cout << "uvel: global_dof = " << global_dof << ", uvel_global_dof = " << vel_global_dof << ", uvel_local_dof = " << vel_local_dof << std::endl;
uvel->replaceLocalValue(vel_local_dof, sol_value);
}
else { // v dof
vel_global_dof = (global_dof-1)/2+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map
vvel->replaceLocalValue(vel_local_dof, sol_value);
}
}
}
else { //note: the case with non-interleaved ordering has not been tested...
int numDofs = overlapMap->getGlobalNumElements();
for (int i=0; i<overlapMap->getNodeNumElements(); i++) {
int global_dof = overlapMap->getGlobalElement(i);
double sol_value = solutionOverlap_constView[i];
int vel_global_dof, vel_local_dof;
if (global_dof < numDofs/2) { //u dof
vel_global_dof = global_dof+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map
uvel->replaceLocalValue(vel_local_dof, sol_value);
}
else { //v dofs
vel_global_dof = global_dof-numDofs/2+1; //add 1 because node_map is 1-based
vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map
vvel->replaceLocalValue(vel_local_dof, sol_value);
}
}
}
#endif
#ifdef WRITE_TO_MATRIX_MARKET
//For debug: write solution to matrix market file
#ifdef CISM_USE_EPETRA
EpetraExt::MultiVectorToMatrixMarketFile("uvel.mm", uvel);
EpetraExt::MultiVectorToMatrixMarketFile("vvel.mm", vvel);
#else
Tpetra_MatrixMarket_Writer::writeDenseFile("uvel.mm", uvel);
Tpetra_MatrixMarket_Writer::writeDenseFile("vvel.mm", vvel);
#endif
#endif
//Copy uvel and vvel into uVel_ptr and vVel_ptr respectively (the arrays passed back to CISM) according to the numbering consistent w/ CISM.
counter1 = 0;
counter2 = 0;
#ifdef CISM_USE_EPETRA
#else
Teuchos::ArrayRCP<const ST> uvel_constView = uvel->get1dView();
Teuchos::ArrayRCP<const ST> vvel_constView = vvel->get1dView();
#endif
local_nodeID = 0;
for (int j=0; j<nsn-1; j++) {
for (int i=0; i<ewn-1; i++) {
for (int k=0; k<upn; k++) {
if (j >= nhalo-1 & j < nsn-nhalo) {
if (i >= nhalo-1 & i < ewn-nhalo) {
#ifdef CISM_USE_EPETRA
local_nodeID = node_map->LID(cismToAlbanyNodeNumberMap[counter1]);
//if (mpiComm->MyPID() == 0)
//std::cout << "counter1:" << counter1 << ", cismToAlbanyNodeNumberMap[counter1]: " << cismToAlbanyNodeNumberMap[counter1] << ", local_nodeID: "
//<< local_nodeID << ", uvel: " << uvel[local_nodeID] << std::endl; //uvel[local_nodeID] << std::endl;
uVel_ptr[counter2] = uvel[local_nodeID];
vVel_ptr[counter2] = vvel[local_nodeID];
#else
local_nodeID = node_map->getLocalElement(cismToAlbanyNodeNumberMap[counter1]);
uVel_ptr[counter2] = uvel_constView[local_nodeID];
vVel_ptr[counter2] = vvel_constView[local_nodeID];
#endif
counter1++;
}
}
else {
uVel_ptr[counter2] = 0.0;
vVel_ptr[counter2] = 0.0;
}
counter2++;
}
}
}
first_time_step = false;
}
void Piro::RythmosSolver<Scalar>::initialize(
const Teuchos::RCP<Teuchos::ParameterList> &appParams,
const Teuchos::RCP< Thyra::ModelEvaluator<Scalar> > &in_model,
const Teuchos::RCP<Rythmos::IntegrationObserverBase<Scalar> > &observer)
{
using Teuchos::ParameterList;
using Teuchos::parameterList;
using Teuchos::RCP;
using Teuchos::rcp;
// set some internals
model = in_model;
num_p = in_model->Np();
num_g = in_model->Ng();
//
*out << "\nA) Get the base parameter list ...\n";
//
RCP<Teuchos::ParameterList> rythmosPL = sublist(appParams, "Rythmos", true);
rythmosPL->validateParameters(*getValidRythmosParameters(),0);
{
const std::string verbosity = rythmosPL->get("Verbosity Level", "VERB_DEFAULT");
if (verbosity == "VERB_NONE") solnVerbLevel = Teuchos::VERB_NONE;
else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT;
else if (verbosity == "VERB_LOW") solnVerbLevel = Teuchos::VERB_LOW;
else if (verbosity == "VERB_MEDIUM") solnVerbLevel = Teuchos::VERB_MEDIUM;
else if (verbosity == "VERB_HIGH") solnVerbLevel = Teuchos::VERB_HIGH;
else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME;
else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Unknown verbosity option specified in Piro_RythmosSolver.");
}
t_initial = rythmosPL->get("Initial Time", 0.0);
t_final = rythmosPL->get("Final Time", 0.1);
const std::string stepperType = rythmosPL->get("Stepper Type", "Backward Euler");
//
*out << "\nC) Create and initalize the forward model ...\n";
//
*out << "\nD) Create the stepper and integrator for the forward problem ...\n";
//
if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "Rythmos") {
Teuchos::RCP<Rythmos::TimeStepNonlinearSolver<Scalar> > rythmosTimeStepSolver =
Rythmos::timeStepNonlinearSolver<Scalar>();
if (rythmosPL->getEntryPtr("NonLinear Solver")) {
RCP<Teuchos::ParameterList> nonlinePL =
sublist(rythmosPL, "NonLinear Solver", true);
rythmosTimeStepSolver->setParameterList(nonlinePL);
}
fwdTimeStepSolver = rythmosTimeStepSolver;
}
else if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "NOX") {
#ifdef Piro_ENABLE_NOX
Teuchos::RCP<Thyra::NOXNonlinearSolver> nox_solver = Teuchos::rcp(new Thyra::NOXNonlinearSolver);
Teuchos::RCP<Teuchos::ParameterList> nox_params = Teuchos::rcp(new Teuchos::ParameterList);
*nox_params = appParams->sublist("NOX");
nox_solver->setParameterList(nox_params);
fwdTimeStepSolver = nox_solver;
#else
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Requested NOX solver for a Rythmos Transient solve, Trilinos was not built with NOX enabled. Please rebuild Trilinos or use the native Rythmos nonlinear solver.");
#endif
}
if (stepperType == "Backward Euler") {
fwdStateStepper = Rythmos::backwardEulerStepper<Scalar> (model, fwdTimeStepSolver);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "Forward Euler") {
fwdStateStepper = Rythmos::forwardEulerStepper<Scalar> (model);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "Explicit RK") {
fwdStateStepper = Rythmos::explicitRKStepper<Scalar>(model);
fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true));
}
else if (stepperType == "BDF") {
Teuchos::RCP<Teuchos::ParameterList> BDFparams =
Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);
Teuchos::RCP<Teuchos::ParameterList> BDFStepControlPL =
Teuchos::sublist(BDFparams,"Step Control Settings");
fwdStateStepper = Teuchos::rcp( new Rythmos::ImplicitBDFStepper<Scalar>(model,fwdTimeStepSolver,BDFparams) );
fwdStateStepper->setInitialCondition(model->getNominalValues());
}
else {
// first (before failing) check to see if the user has added stepper factory
typename std::map<std::string,Teuchos::RCP<RythmosStepperFactory<Scalar> > >::const_iterator
stepFactItr = stepperFactories.find(stepperType);
if(stepFactItr!=stepperFactories.end()) {
// the user has added it, hot dog lets build a new stepper!
Teuchos::RCP<Teuchos::ParameterList> stepperParams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true);
// build the stepper using the factory
fwdStateStepper = stepFactItr->second->buildStepper(model,fwdTimeStepSolver,stepperParams);
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(
true, Teuchos::Exceptions::InvalidParameter,
std::endl << "Error! Piro::Epetra::RythmosSolver: Invalid Steper Type: "
<< stepperType << std::endl);
}
}
// Step control strategy
{
// If the stepper can accept a step control strategy, then attempt to build one.
RCP<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> > scsa_stepper =
Teuchos::rcp_dynamic_cast<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> >(fwdStateStepper);
if (Teuchos::nonnull(scsa_stepper)) {
const std::string step_control_strategy = rythmosPL->get("Step Control Strategy Type", "None");
if (step_control_strategy == "None") {
// don't do anything, stepper will build default
} else if (step_control_strategy == "ImplicitBDFRamping") {
const RCP<Rythmos::ImplicitBDFStepperRampingStepControl<Scalar> > rscs =
rcp(new Rythmos::ImplicitBDFStepperRampingStepControl<Scalar>);
const RCP<ParameterList> p = parameterList(rythmosPL->sublist("Rythmos Step Control Strategy"));
rscs->setParameterList(p);
scsa_stepper->setStepControlStrategy(rscs);
} else {
TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
"Error! Piro::Epetra::RythmosSolver: Invalid step control strategy type: "
<< step_control_strategy << std::endl);
}
}
}
{
const RCP<Teuchos::ParameterList> integrationControlPL =
Teuchos::sublist(rythmosPL, "Rythmos Integration Control", true);
RCP<Rythmos::DefaultIntegrator<Scalar> > defaultIntegrator;
if (rythmosPL->get("Rythmos Integration Control Strategy", "Simple") == "Simple") {
defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL));
}
else if(rythmosPL->get<std::string>("Rythmos Integration Control Strategy") == "Ramping") {
defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::rampingIntegrationControlStrategy<Scalar>(integrationControlPL));
}
fwdStateIntegrator = defaultIntegrator;
}
fwdStateIntegrator->setParameterList(sublist(rythmosPL, "Rythmos Integrator", true));
if (Teuchos::nonnull(observer)) {
fwdStateIntegrator->setIntegrationObserver(observer);
}
isInitialized = true;
}