Piro::LOCASolver<Scalar>::LOCASolver( const Teuchos::RCP<Teuchos::ParameterList> &piroParams, const Teuchos::RCP<Thyra::ModelEvaluator<Scalar> > &model, const Teuchos::RCP<LOCA::Thyra::SaveDataStrategy> &saveDataStrategy) : SteadyStateSolver<Scalar>(model, model->Np() > 0), // Only one parameter supported piroParams_(piroParams), saveDataStrategy_(saveDataStrategy), globalData_(LOCA::createGlobalData(piroParams)), paramVector_(), group_(), locaStatusTests_(), noxStatusTests_(), stepper_() { const int l = 0; // TODO: Allow user to select parameter index const Detail::ModelEvaluatorParamName paramName(this->getModel().get_p_names(l)); const Thyra::Ordinal p_entry_count = this->getModel().get_p_space(l)->dim(); for (Teuchos_Ordinal k = 0; k < p_entry_count; ++k) { (void) paramVector_.addParameter(paramName(k)); } const NOX::Thyra::Vector initialGuess(*model->getNominalValues().get_x()); group_ = Teuchos::rcp(new LOCA::Thyra::Group(globalData_, initialGuess, model, paramVector_, l)); group_->setSaveDataStrategy(saveDataStrategy_); // TODO: Create non-trivial stopping criterion for the stepper locaStatusTests_ = Teuchos::null; // Create stopping criterion for the nonlinear solver const Teuchos::RCP<Teuchos::ParameterList> noxStatusParams = Teuchos::sublist(Teuchos::sublist(piroParams_, "NOX"), "Status Tests"); noxStatusTests_ = NOX::StatusTest::buildStatusTests(*noxStatusParams, *(globalData_->locaUtils)); stepper_ = Teuchos::rcp(new LOCA::Stepper(globalData_, group_, locaStatusTests_, noxStatusTests_, piroParams_)); }
Piro::LOCAAdaptiveSolver<Scalar>::LOCAAdaptiveSolver( const Teuchos::RCP<Teuchos::ParameterList> &piroParams, const Teuchos::RCP<Thyra::ModelEvaluator<Scalar> > &model, const Teuchos::RCP<Thyra::AdaptiveSolutionManager> &solMgr, const Teuchos::RCP<LOCA::Thyra::SaveDataStrategy> &saveDataStrategy) : SteadyStateSolver<Scalar>(model, model->Np() > 0), // Only one parameter supported piroParams_(piroParams), saveDataStrategy_(saveDataStrategy), globalData_(LOCA::createGlobalData(piroParams)), paramVector_(), solMgr_(solMgr), locaStatusTests_(), noxStatusTests_(), stepper_() { const int l = 0; // TODO: Allow user to select parameter index const Detail::ModelEvaluatorParamName paramName(this->getModel().get_p_names(l)); const Thyra::Ordinal p_entry_count = this->getModel().get_p_space(l)->dim(); for (Teuchos_Ordinal k = 0; k < p_entry_count; ++k) { (void) paramVector_.addParameter(paramName(k)); } solMgr_->initialize(Teuchos::rcp(new Thyra::LOCAAdaptiveState(model, saveDataStrategy_, globalData_, Teuchos::rcpFromRef(paramVector_), l))); // TODO: Create non-trivial stopping criterion for the stepper locaStatusTests_ = Teuchos::null; // Create stopping criterion for the nonlinear solver const Teuchos::RCP<Teuchos::ParameterList> noxStatusParams = Teuchos::sublist(Teuchos::sublist(piroParams_, "NOX"), "Status Tests"); noxStatusTests_ = NOX::StatusTest::buildStatusTests(*noxStatusParams, *(globalData_->locaUtils)); stepper_ = Teuchos::rcp(new LOCA::AdaptiveStepper(piroParams_, solMgr_, globalData_, noxStatusTests_)); if (piroParams_->isSublist("NOX") && piroParams_->sublist("NOX").isSublist("Printing")) utils_.reset(piroParams_->sublist("NOX").sublist("Printing")); }
Piro::SteadyStateSolver<Scalar>:: SteadyStateSolver(const Teuchos::RCP<const Thyra::ModelEvaluator<Scalar> > &model) : model_(model), num_p_(model->Np()), num_g_(model->Ng()) {}
// The solve is done in the felix_driver_run function, and the solution is passed back to Glimmer-CISM // IK, 12/3/13: time_inc_yr and cur_time_yr are not used here... void felix_driver_run(FelixToGlimmer * ftg_ptr, double& cur_time_yr, double time_inc_yr) { //IK, 12/9/13: how come FancyOStream prints an all processors?? Teuchos::RCP<Teuchos::FancyOStream> out(Teuchos::VerboseObjectBase::getDefaultOStream()); if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0) { std::cout << "In felix_driver_run, cur_time, time_inc = " << cur_time_yr << " " << time_inc_yr << std::endl; } // --------------------------------------------- // get u and v velocity solution from Glimmer-CISM // IK, 11/26/13: need to concatenate these into a single solve for initial condition for Albany/FELIX solve // IK, 3/14/14: moved this step to felix_driver_run from felix_driver init, since we still want to grab and u and v velocities for CISM if the mesh hasn't changed, // in which case only felix_driver_run will be called, not felix_driver_init. // --------------------------------------------- if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0) std::cout << "In felix_driver_run: grabbing pointers to u and v velocities in CISM..." << std::endl; uVel_ptr = ftg_ptr ->getDoubleVar("uvel", "velocity"); vVel_ptr = ftg_ptr ->getDoubleVar("vvel", "velocity"); // --------------------------------------------- // Set restart solution to the one passed from CISM // IK, 3/14/14: moved this from felix_driver_init to felix_driver_run. // --------------------------------------------- if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0) std::cout << "In felix_driver_run: setting initial condition from CISM..." << std::endl; //Check what kind of ordering you have in the solution & create solutionField object. interleavedOrdering = meshStruct->getInterleavedOrdering(); Albany::AbstractSTKFieldContainer::VectorFieldType* solutionField; if(interleavedOrdering) solutionField = Teuchos::rcp_dynamic_cast<Albany::OrdinarySTKFieldContainer<true> >(meshStruct->getFieldContainer())->getSolutionField(); else solutionField = Teuchos::rcp_dynamic_cast<Albany::OrdinarySTKFieldContainer<false> >(meshStruct->getFieldContainer())->getSolutionField(); //Create vector used to renumber nodes on each processor from the Albany convention (horizontal levels first) to the CISM convention (vertical layers first) nNodes2D = (global_ewn + 1)*(global_nsn+1); //number global nodes in the domain in 2D nNodesProc2D = (nsn-2*nhalo+1)*(ewn-2*nhalo+1); //number of nodes on each processor in 2D cismToAlbanyNodeNumberMap.resize(upn*nNodesProc2D); for (int j=0; j<nsn-2*nhalo+1;j++) { for (int i=0; i<ewn-2*nhalo+1; i++) { for (int k=0; k<upn; k++) { int index = k+upn*i + j*(ewn-2*nhalo+1)*upn; cismToAlbanyNodeNumberMap[index] = k*nNodes2D + global_node_id_owned_map_Ptr[i+j*(ewn-2*nhalo+1)]; //if (mpiComm->MyPID() == 0) // std::cout << "index: " << index << ", cismToAlbanyNodeNumberMap: " << cismToAlbanyNodeNumberMap[index] << std::endl; } } } //The way it worked out, uVel_ptr and vVel_ptr have more nodes than the nodes in the mesh passed to Albany/CISM for the solve. In particular, //there is 1 row of halo elements in uVel_ptr and vVel_ptr. To account for this, we copy uVel_ptr and vVel_ptr into std::vectors, which do not have the halo elements. std::vector<double> uvel_vec(upn*nNodesProc2D); std::vector<double> vvel_vec(upn*nNodesProc2D); int counter1 = 0; int counter2 = 0; int local_nodeID; for (int j=0; j<nsn-1; j++) { for (int i=0; i<ewn-1; i++) { for (int k=0; k<upn; k++) { if (j >= nhalo-1 & j < nsn-nhalo) { if (i >= nhalo-1 & i < ewn-nhalo) { #ifdef CISM_USE_EPETRA local_nodeID = node_map->LID(cismToAlbanyNodeNumberMap[counter1]); #else local_nodeID = node_map->getLocalElement(cismToAlbanyNodeNumberMap[counter1]); #endif uvel_vec[counter1] = uVel_ptr[counter2]; vvel_vec[counter1] = vVel_ptr[counter2]; counter1++; } } counter2++; } } } //Loop over all the elements to find which nodes are active. For the active nodes, copy uvel and vvel from CISM into Albany solution array to //use as initial condition. //NOTE: there is some inefficiency here by looping over all the elements. TO DO? pass only active nodes from Albany-CISM to improve this? double velScale = seconds_per_year*vel_scaling_param; for (int i=0; i<nElementsActive; i++) { for (int j=0; j<8; j++) { int node_GID = global_element_conn_active_Ptr[i + nElementsActive*j]; //node_GID is 1-based #ifdef CISM_USE_EPETRA int node_LID = node_map->LID(node_GID); //node_LID is 0-based #else int node_LID = node_map->getLocalElement(node_GID); //node_LID is 0-based #endif stk::mesh::Entity node = meshStruct->bulkData->get_entity(stk::topology::NODE_RANK, node_GID); double* sol = stk::mesh::field_data(*solutionField, node); //IK, 3/18/14: added division by velScale to convert uvel and vvel from dimensionless to having units of m/year (the Albany units) sol[0] = uvel_vec[node_LID]/velScale; sol[1] = vvel_vec[node_LID]/velScale; } } // --------------------------------------------------------------------------------------------------- // Solve // --------------------------------------------------------------------------------------------------- if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0) std::cout << "In felix_driver_run: starting the solve... " << std::endl; //Need to set HasRestart solution such that uvel_Ptr and vvel_Ptr (u and v from Glimmer/CISM) are always set as initial condition? meshStruct->setHasRestartSolution(!first_time_step); //Turn off homotopy if we're not in the first time-step. //NOTE - IMPORTANT: Glen's Law Homotopy parameter should be set to 1.0 in the parameter list for this logic to work!!! if (!first_time_step) { meshStruct->setRestartDataTime(parameterList->sublist("Problem").get("Homotopy Restart Step", 1.)); double homotopy = parameterList->sublist("Problem").sublist("FELIX Viscosity").get("Glen's Law Homotopy Parameter", 1.0); if(meshStruct->restartDataTime()== homotopy) { parameterList->sublist("Problem").set("Solution Method", "Steady"); parameterList->sublist("Piro").set("Solver Type", "NOX"); } } albanyApp->createDiscretization(); //IK, 10/30/14: Check that # of elements from previous time step hasn't changed. //If it has not, use previous solution as initial guess for current time step. //Otherwise do not set initial solution. It's possible this can be improved so some part of the previous solution is used //defined on the current mesh (if it receded, which likely it will in dynamic ice sheet simulations...). if (nElementsActivePrevious != nElementsActive) previousSolution = Teuchos::null; albanyApp->finalSetUp(parameterList, previousSolution); //if (!first_time_step) // std::cout << "previousSolution: " << *previousSolution << std::endl; #ifdef CISM_USE_EPETRA solver = slvrfctry->createThyraSolverAndGetAlbanyApp(albanyApp, mpiComm, mpiComm, Teuchos::null, false); #else solver = slvrfctry->createAndGetAlbanyAppT(albanyApp, mpiCommT, mpiCommT, Teuchos::null, false); #endif Teuchos::ParameterList solveParams; solveParams.set("Compute Sensitivities", true); Teuchos::Array<Teuchos::RCP<const Thyra::VectorBase<double> > > thyraResponses; Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Thyra::MultiVectorBase<double> > > > thyraSensitivities; Piro::PerformSolveBase(*solver, solveParams, thyraResponses, thyraSensitivities); #ifdef CISM_USE_EPETRA const Epetra_Map& ownedMap(*albanyApp->getDiscretization()->getMap()); //owned map const Epetra_Map& overlapMap(*albanyApp->getDiscretization()->getOverlapMap()); //overlap map Epetra_Import import(overlapMap, ownedMap); //importer from ownedMap to overlapMap Epetra_Vector solutionOverlap(overlapMap); //overlapped solution solutionOverlap.Import(*albanyApp->getDiscretization()->getSolutionField(), import, Insert); #else Teuchos::RCP<const Tpetra_Map> ownedMap = albanyApp->getDiscretization()->getMapT(); //owned map Teuchos::RCP<const Tpetra_Map> overlapMap = albanyApp->getDiscretization()->getOverlapMapT(); //overlap map Teuchos::RCP<Tpetra_Import> import = Teuchos::rcp(new Tpetra_Import(ownedMap, overlapMap)); Teuchos::RCP<Tpetra_Vector> solutionOverlap = Teuchos::rcp(new Tpetra_Vector(overlapMap)); solutionOverlap->doImport(*albanyApp->getDiscretization()->getSolutionFieldT(), *import, Tpetra::INSERT); Teuchos::ArrayRCP<const ST> solutionOverlap_constView = solutionOverlap->get1dView(); #endif #ifdef WRITE_TO_MATRIX_MARKET #ifdef CISM_USE_EPETRA //For debug: write solution and maps to matrix market file EpetraExt::BlockMapToMatrixMarketFile("node_map.mm", *node_map); EpetraExt::BlockMapToMatrixMarketFile("map.mm", ownedMap); EpetraExt::BlockMapToMatrixMarketFile("overlap_map.mm", overlapMap); EpetraExt::MultiVectorToMatrixMarketFile("solution.mm", *albanyApp->getDiscretization()->getSolutionField()); #else Tpetra_MatrixMarket_Writer::writeMapFile("node_map.mm", *node_map); Tpetra_MatrixMarket_Writer::writeMapFile("map.mm", *ownedMap); Tpetra_MatrixMarket_Writer::writeMapFile("overlap_map.mm", *overlapMap); Tpetra_MatrixMarket_Writer::writeDenseFile("solution.mm", app->getDiscretization()->getSolutionFieldT()); #endif #endif //set previousSolution (used as initial guess for next time step) to final Albany solution. previousSolution = Teuchos::rcp(new Tpetra_Vector(*albanyApp->getDiscretization()->getSolutionFieldT())); nElementsActivePrevious = nElementsActive; //std::cout << "Final solution: " << *albanyApp->getDiscretization()->getSolutionField() << std::endl; // --------------------------------------------------------------------------------------------------- // Compute sensitivies / responses and perform regression tests // IK, 12/9/13: how come this is turned off in mpas branch? // --------------------------------------------------------------------------------------------------- if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0) std::cout << "Computing responses and sensitivities..." << std::endl; int status=0; // 0 = pass, failures are incremented #ifdef CISM_USE_EPETRA Teuchos::Array<Teuchos::RCP<const Epetra_Vector> > responses; Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Epetra_MultiVector> > > sensitivities; epetraFromThyra(mpiComm, thyraResponses, thyraSensitivities, responses, sensitivities); #else Teuchos::Array<Teuchos::RCP<const Tpetra_Vector> > responses; Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Tpetra_MultiVector> > > sensitivities; tpetraFromThyra(thyraResponses, thyraSensitivities, responses, sensitivities); #endif const int num_p = solver->Np(); // Number of *vectors* of parameters const int num_g = solver->Ng(); // Number of *vectors* of responses if (debug_output_verbosity != 0) { *out << "Finished eval of first model: Params, Responses " << std::setprecision(12) << std::endl; } const Thyra::ModelEvaluatorBase::InArgs<double> nominal = solver->getNominalValues(); if (debug_output_verbosity != 0) { for (int i=0; i<num_p; i++) { #ifdef CISM_USE_EPETRA const Teuchos::RCP<const Epetra_Vector> p_init = epetraVectorFromThyra(mpiComm, nominal.get_p(i)); p_init->Print(*out << "\nParameter vector " << i << ":\n"); #else Albany::printTpetraVector(*out << "\nParameter vector " << i << ":\n", ConverterT::getConstTpetraVector(nominal.get_p(i))); #endif } } for (int i=0; i<num_g-1; i++) { #ifdef CISM_USE_EPETRA const Teuchos::RCP<const Epetra_Vector> g = responses[i]; #else const Teuchos::RCP<const Tpetra_Vector> g = responses[i]; #endif bool is_scalar = true; if (albanyApp != Teuchos::null) is_scalar = albanyApp->getResponse(i)->isScalarResponse(); if (is_scalar) { if (debug_output_verbosity != 0) { #ifdef CISM_USE_EPETRA g->Print(*out << "\nResponse vector " << i << ":\n"); #else Albany::printTpetraVector(*out << "\nResponse vector " << i << ":\n", g); #endif } if (num_p == 0 && cur_time_yr == final_time) { // Just calculate regression data -- only if in final time step #ifdef CISM_USE_EPETRA status += slvrfctry->checkSolveTestResults(i, 0, g.get(), NULL); #else status += slvrfctry->checkSolveTestResultsT(i, 0, g.get(), NULL); #endif } else { for (int j=0; j<num_p; j++) { #ifdef CISM_USE_EPETRA const Teuchos::RCP<const Epetra_MultiVector> dgdp = sensitivities[i][j]; #else const Teuchos::RCP<const Tpetra_MultiVector> dgdp = sensitivities[i][j]; #endif if (debug_output_verbosity != 0) { if (Teuchos::nonnull(dgdp)) { #ifdef CISM_USE_EPETRA dgdp->Print(*out << "\nSensitivities (" << i << "," << j << "):!\n"); #else Albany::printTpetraVector(*out << "\nSensitivities (" << i << "," << j << "):!\n", dgdp); #endif } } if (cur_time_yr == final_time) { #ifdef CISM_USE_EPETRA status += slvrfctry->checkSolveTestResults(i, j, g.get(), dgdp.get()); #else status += slvrfctry->checkSolveTestResultsT(i, j, g.get(), dgdp.get()); #endif } } } } } if (debug_output_verbosity != 0 && cur_time_yr == final_time) //only print regression test result if you're in the final time step *out << "\nNumber of Failed Comparisons: " << status << std::endl; //IK, 10/30/14: added the following line so that when you run ctest from CISM the test fails if there are some failed comparisons. if (status > 0) TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, "All regression comparisons did not pass!" << std::endl); // --------------------------------------------------------------------------------------------------- // Copy solution back to glimmer uvel and vvel arrays to be passed back // --------------------------------------------------------------------------------------------------- //std::cout << "overlapMap # global elements: " << overlapMap.NumGlobalElements() << std::endl; //std::cout << "overlapMap # my elements: " << overlapMap.NumMyElements() << std::endl; //std::cout << "overlapMap: " << overlapMap << std::endl; //std::cout << "map # global elements: " << ownedMap.NumGlobalElements() << std::endl; //std::cout << "map # my elements: " << ownedMap.NumMyElements() << std::endl; //std::cout << "node_map # global elements: " << node_map->NumGlobalElements() << std::endl; //std::cout << "node_map # my elements: " << node_map->NumMyElements() << std::endl; //std::cout << "node_map: " << *node_map << std::endl; if (debug_output_verbosity != 0 & mpiCommT->getRank() == 0) std::cout << "In felix_driver_run: copying Albany solution to uvel and vvel to send back to CISM... " << std::endl; #ifdef CISM_USE_EPETRA //Epetra_Vectors to hold uvel and vvel to be passed to Glimmer/CISM Epetra_Vector uvel(*node_map, true); Epetra_Vector vvel(*node_map, true); #else //Tpetra_Vectors to hold uvel and vvel to be passed to Glimmer/CISM Teuchos::RCP<Tpetra_Vector> uvel = Teuchos::rcp(new Tpetra_Vector(node_map, true)); Teuchos::RCP<Tpetra_Vector> vvel = Teuchos::rcp(new Tpetra_Vector(node_map, true)); #endif #ifdef CISM_USE_EPETRA if (interleavedOrdering == true) { for (int i=0; i<overlapMap.NumMyElements(); i++) { int global_dof = overlapMap.GID(i); double sol_value = solutionOverlap[i]; int modulo = (global_dof % 2); //check if dof is for u or for v int vel_global_dof, vel_local_dof; if (modulo == 0) { //u dof vel_global_dof = global_dof/2+1; //add 1 because node_map is 1-based vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map //std::cout << "uvel: global_dof = " << global_dof << ", uvel_global_dof = " << vel_global_dof << ", uvel_local_dof = " << vel_local_dof << std::endl; uvel.ReplaceMyValues(1, &sol_value, &vel_local_dof); } else { // v dof vel_global_dof = (global_dof-1)/2+1; //add 1 because node_map is 1-based vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map vvel.ReplaceMyValues(1, &sol_value, & vel_local_dof); } } } else { //note: the case with non-interleaved ordering has not been tested... int numDofs = overlapMap.NumGlobalElements(); for (int i=0; i<overlapMap.NumMyElements(); i++) { int global_dof = overlapMap.GID(i); double sol_value = solutionOverlap[i]; int vel_global_dof, vel_local_dof; if (global_dof < numDofs/2) { //u dof vel_global_dof = global_dof+1; //add 1 because node_map is 1-based vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map uvel.ReplaceMyValues(1, &sol_value, &vel_local_dof); } else { //v dofs vel_global_dof = global_dof-numDofs/2+1; //add 1 because node_map is 1-based vel_local_dof = node_map->LID(vel_global_dof); //look up local id corresponding to global id in node_map vvel.ReplaceMyValues(1, &sol_value, & vel_local_dof); } } } #else if (interleavedOrdering == true) { for (int i=0; i<overlapMap->getNodeNumElements(); i++) { int global_dof = overlapMap->getGlobalElement(i); double sol_value = solutionOverlap_constView[i]; int modulo = (global_dof % 2); //check if dof is for u or for v int vel_global_dof, vel_local_dof; if (modulo == 0) { //u dof vel_global_dof = global_dof/2+1; //add 1 because node_map is 1-based vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map //std::cout << "uvel: global_dof = " << global_dof << ", uvel_global_dof = " << vel_global_dof << ", uvel_local_dof = " << vel_local_dof << std::endl; uvel->replaceLocalValue(vel_local_dof, sol_value); } else { // v dof vel_global_dof = (global_dof-1)/2+1; //add 1 because node_map is 1-based vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map vvel->replaceLocalValue(vel_local_dof, sol_value); } } } else { //note: the case with non-interleaved ordering has not been tested... int numDofs = overlapMap->getGlobalNumElements(); for (int i=0; i<overlapMap->getNodeNumElements(); i++) { int global_dof = overlapMap->getGlobalElement(i); double sol_value = solutionOverlap_constView[i]; int vel_global_dof, vel_local_dof; if (global_dof < numDofs/2) { //u dof vel_global_dof = global_dof+1; //add 1 because node_map is 1-based vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map uvel->replaceLocalValue(vel_local_dof, sol_value); } else { //v dofs vel_global_dof = global_dof-numDofs/2+1; //add 1 because node_map is 1-based vel_local_dof = node_map->getLocalElement(vel_global_dof); //look up local id corresponding to global id in node_map vvel->replaceLocalValue(vel_local_dof, sol_value); } } } #endif #ifdef WRITE_TO_MATRIX_MARKET //For debug: write solution to matrix market file #ifdef CISM_USE_EPETRA EpetraExt::MultiVectorToMatrixMarketFile("uvel.mm", uvel); EpetraExt::MultiVectorToMatrixMarketFile("vvel.mm", vvel); #else Tpetra_MatrixMarket_Writer::writeDenseFile("uvel.mm", uvel); Tpetra_MatrixMarket_Writer::writeDenseFile("vvel.mm", vvel); #endif #endif //Copy uvel and vvel into uVel_ptr and vVel_ptr respectively (the arrays passed back to CISM) according to the numbering consistent w/ CISM. counter1 = 0; counter2 = 0; #ifdef CISM_USE_EPETRA #else Teuchos::ArrayRCP<const ST> uvel_constView = uvel->get1dView(); Teuchos::ArrayRCP<const ST> vvel_constView = vvel->get1dView(); #endif local_nodeID = 0; for (int j=0; j<nsn-1; j++) { for (int i=0; i<ewn-1; i++) { for (int k=0; k<upn; k++) { if (j >= nhalo-1 & j < nsn-nhalo) { if (i >= nhalo-1 & i < ewn-nhalo) { #ifdef CISM_USE_EPETRA local_nodeID = node_map->LID(cismToAlbanyNodeNumberMap[counter1]); //if (mpiComm->MyPID() == 0) //std::cout << "counter1:" << counter1 << ", cismToAlbanyNodeNumberMap[counter1]: " << cismToAlbanyNodeNumberMap[counter1] << ", local_nodeID: " //<< local_nodeID << ", uvel: " << uvel[local_nodeID] << std::endl; //uvel[local_nodeID] << std::endl; uVel_ptr[counter2] = uvel[local_nodeID]; vVel_ptr[counter2] = vvel[local_nodeID]; #else local_nodeID = node_map->getLocalElement(cismToAlbanyNodeNumberMap[counter1]); uVel_ptr[counter2] = uvel_constView[local_nodeID]; vVel_ptr[counter2] = vvel_constView[local_nodeID]; #endif counter1++; } } else { uVel_ptr[counter2] = 0.0; vVel_ptr[counter2] = 0.0; } counter2++; } } } first_time_step = false; }
void Piro::RythmosSolver<Scalar>::initialize( #endif const Teuchos::RCP<Teuchos::ParameterList> &appParams, const Teuchos::RCP< Thyra::ModelEvaluator<Scalar> > &in_model, const Teuchos::RCP<Rythmos::IntegrationObserverBase<Scalar> > &observer) { using Teuchos::ParameterList; using Teuchos::parameterList; using Teuchos::RCP; using Teuchos::rcp; // set some internals model = in_model; num_p = in_model->Np(); num_g = in_model->Ng(); // *out << "\nA) Get the base parameter list ...\n"; // if (appParams->isSublist("Rythmos")) { RCP<Teuchos::ParameterList> rythmosPL = sublist(appParams, "Rythmos", true); rythmosPL->validateParameters(*getValidRythmosParameters(),0); { const std::string verbosity = rythmosPL->get("Verbosity Level", "VERB_DEFAULT"); if (verbosity == "VERB_NONE") solnVerbLevel = Teuchos::VERB_NONE; else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT; else if (verbosity == "VERB_LOW") solnVerbLevel = Teuchos::VERB_LOW; else if (verbosity == "VERB_MEDIUM") solnVerbLevel = Teuchos::VERB_MEDIUM; else if (verbosity == "VERB_HIGH") solnVerbLevel = Teuchos::VERB_HIGH; else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME; else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Unknown verbosity option specified in Piro_RythmosSolver."); } t_initial = rythmosPL->get("Initial Time", 0.0); t_final = rythmosPL->get("Final Time", 0.1); const std::string stepperType = rythmosPL->get("Stepper Type", "Backward Euler"); // *out << "\nC) Create and initalize the forward model ...\n"; // *out << "\nD) Create the stepper and integrator for the forward problem ...\n"; // if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "Rythmos") { Teuchos::RCP<Rythmos::TimeStepNonlinearSolver<Scalar> > rythmosTimeStepSolver = Rythmos::timeStepNonlinearSolver<Scalar>(); if (rythmosPL->getEntryPtr("NonLinear Solver")) { RCP<Teuchos::ParameterList> nonlinePL = sublist(rythmosPL, "NonLinear Solver", true); rythmosTimeStepSolver->setParameterList(nonlinePL); } fwdTimeStepSolver = rythmosTimeStepSolver; } else if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "NOX") { #ifdef HAVE_PIRO_NOX Teuchos::RCP<Thyra::NOXNonlinearSolver> nox_solver = Teuchos::rcp(new Thyra::NOXNonlinearSolver); Teuchos::RCP<Teuchos::ParameterList> nox_params = Teuchos::rcp(new Teuchos::ParameterList); *nox_params = appParams->sublist("NOX"); nox_solver->setParameterList(nox_params); fwdTimeStepSolver = nox_solver; #else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Requested NOX solver for a Rythmos Transient solve, Trilinos was not built with NOX enabled. Please rebuild Trilinos or use the native Rythmos nonlinear solver."); #endif } if (stepperType == "Backward Euler") { fwdStateStepper = Rythmos::backwardEulerStepper<Scalar> (model, fwdTimeStepSolver); fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true)); } else if (stepperType == "Forward Euler") { fwdStateStepper = Rythmos::forwardEulerStepper<Scalar> (model); fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true)); } else if (stepperType == "Explicit RK") { fwdStateStepper = Rythmos::explicitRKStepper<Scalar>(model); fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true)); } else if (stepperType == "BDF") { Teuchos::RCP<Teuchos::ParameterList> BDFparams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true); Teuchos::RCP<Teuchos::ParameterList> BDFStepControlPL = Teuchos::sublist(BDFparams,"Step Control Settings"); fwdStateStepper = Teuchos::rcp( new Rythmos::ImplicitBDFStepper<Scalar>(model,fwdTimeStepSolver,BDFparams) ); fwdStateStepper->setInitialCondition(model->getNominalValues()); } else { // first (before failing) check to see if the user has added stepper factory typename std::map<std::string,Teuchos::RCP<Piro::RythmosStepperFactory<Scalar> > >::const_iterator stepFactItr = stepperFactories.find(stepperType); if(stepFactItr!=stepperFactories.end()) { // the user has added it, hot dog lets build a new stepper! Teuchos::RCP<Teuchos::ParameterList> stepperParams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true); // build the stepper using the factory fwdStateStepper = stepFactItr->second->buildStepper(model,fwdTimeStepSolver,stepperParams); // the user decided to override the model being used (let them) if(fwdStateStepper->getModel()!=model && fwdStateStepper->getModel()!=Teuchos::null) { model = Teuchos::rcp_const_cast<Thyra::ModelEvaluator<Scalar> >(fwdStateStepper->getModel()); num_p = in_model->Np(); num_g = in_model->Ng(); } } else { TEUCHOS_TEST_FOR_EXCEPTION( true, Teuchos::Exceptions::InvalidParameter, std::endl << "Error! Piro::RythmosSolver: Invalid Steper Type: " << stepperType << std::endl); } } // Step control strategy { // If the stepper can accept a step control strategy, then attempt to build one. RCP<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> > scsa_stepper = Teuchos::rcp_dynamic_cast<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> >(fwdStateStepper); if (Teuchos::nonnull(scsa_stepper)) { const std::string step_control_strategy = rythmosPL->get("Step Control Strategy Type", "None"); if (step_control_strategy == "None") { // don't do anything, stepper will build default } else if (step_control_strategy == "ImplicitBDFRamping") { const RCP<Rythmos::ImplicitBDFStepperRampingStepControl<Scalar> > rscs = rcp(new Rythmos::ImplicitBDFStepperRampingStepControl<Scalar>); const RCP<ParameterList> p = parameterList(rythmosPL->sublist("Rythmos Step Control Strategy")); rscs->setParameterList(p); scsa_stepper->setStepControlStrategy(rscs); } else { // first (before failing) check to see if the user has added step control factory typename std::map<std::string,Teuchos::RCP<Piro::RythmosStepControlFactory<Scalar> > >::const_iterator stepControlFactItr = stepControlFactories.find(step_control_strategy); if (stepControlFactItr != stepControlFactories.end()) { const RCP<Rythmos::StepControlStrategyBase<Scalar> > rscs = stepControlFactItr->second->buildStepControl(); const RCP<ParameterList> p = parameterList(rythmosPL -> sublist("Rythmos Step Control Strategy")); rscs->setParameterList(p); scsa_stepper->setStepControlStrategy(rscs); } else { TEUCHOS_TEST_FOR_EXCEPTION( true, std::logic_error, "Error! Piro::RythmosSolver: Invalid step control strategy type: " << step_control_strategy << std::endl); } } } } { const RCP<Teuchos::ParameterList> integrationControlPL = Teuchos::sublist(rythmosPL, "Rythmos Integration Control", true); RCP<Rythmos::DefaultIntegrator<Scalar> > defaultIntegrator; if (rythmosPL->get("Rythmos Integration Control Strategy", "Simple") == "Simple") { defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL)); } else if(rythmosPL->get<std::string>("Rythmos Integration Control Strategy") == "Ramping") { defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::rampingIntegrationControlStrategy<Scalar>(integrationControlPL)); } fwdStateIntegrator = defaultIntegrator; } fwdStateIntegrator->setParameterList(sublist(rythmosPL, "Rythmos Integrator", true)); if (Teuchos::nonnull(observer)) { fwdStateIntegrator->setIntegrationObserver(observer); } } else if (appParams->isSublist("Rythmos Solver")) { /** New parameter list format **/ RCP<Teuchos::ParameterList> rythmosSolverPL = sublist(appParams, "Rythmos Solver", true); RCP<Teuchos::ParameterList> rythmosPL = sublist(rythmosSolverPL, "Rythmos", true); { const std::string verbosity = rythmosSolverPL->get("Verbosity Level", "VERB_DEFAULT"); if (verbosity == "VERB_NONE") solnVerbLevel = Teuchos::VERB_NONE; else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT; else if (verbosity == "VERB_LOW") solnVerbLevel = Teuchos::VERB_LOW; else if (verbosity == "VERB_MEDIUM") solnVerbLevel = Teuchos::VERB_MEDIUM; else if (verbosity == "VERB_HIGH") solnVerbLevel = Teuchos::VERB_HIGH; else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME; else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, "Unknown verbosity option specified in Piro_RythmosSolver."); } t_initial = rythmosPL->sublist("Integrator Settings").get("Initial Time", 0.0); t_final = rythmosPL->sublist("Integrator Settings").get("Final Time", 0.1); const std::string stepperType = rythmosPL->sublist("Stepper Settings") .sublist("Stepper Selection").get("Stepper Type", "Backward Euler"); // // *out << "\nB) Create the Stratimikos linear solver factory ...\n"; // // This is the linear solve strategy that will be used to solve for the // linear system with the W. // Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder; #ifdef HAVE_PIRO_IFPACK2 typedef Thyra::PreconditionerFactoryBase<double> Base; #ifdef ALBANY_BUILD typedef Thyra::Ifpack2PreconditionerFactory<Tpetra::CrsMatrix<double, LocalOrdinal, GlobalOrdinal, Node> > Impl; #else typedef Thyra::Ifpack2PreconditionerFactory<Tpetra::CrsMatrix<double> > Impl; #endif linearSolverBuilder.setPreconditioningStrategyFactory(Teuchos::abstractFactoryStd<Base, Impl>(), "Ifpack2"); #endif #ifdef HAVE_PIRO_MUELU #ifdef ALBANY_BUILD Stratimikos::enableMueLu<LocalOrdinal, GlobalOrdinal, Node>(linearSolverBuilder); #else Stratimikos::enableMueLu(linearSolverBuilder); #endif #endif linearSolverBuilder.setParameterList(sublist(rythmosSolverPL, "Stratimikos", true)); rythmosSolverPL->validateParameters(*getValidRythmosSolverParameters(),0); RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory = createLinearSolveStrategy(linearSolverBuilder); // *out << "\nC) Create and initalize the forward model ...\n"; // // C.1) Create the underlying EpetraExt::ModelEvaluator // already constructed as "model". Decorate if needed. // TODO: Generelize to any explicit method, option to invert mass matrix if (stepperType == "Explicit RK") { if (rythmosSolverPL->get("Invert Mass Matrix", false)) { Teuchos::RCP<Thyra::ModelEvaluator<Scalar> > origModel = model; rythmosSolverPL->get("Lump Mass Matrix", false); //JF line does not do anything model = Teuchos::rcp(new Piro::InvertMassMatrixDecorator<Scalar>( sublist(rythmosSolverPL,"Stratimikos", true), origModel, true,rythmosSolverPL->get("Lump Mass Matrix", false),false)); } } // C.2) Create the Thyra-wrapped ModelEvaluator thyraModel = rcp(new Thyra::DefaultModelEvaluatorWithSolveFactory<Scalar>(model, lowsFactory)); const RCP<const Thyra::VectorSpaceBase<double> > x_space = thyraModel->get_x_space(); // *out << "\nD) Create the stepper and integrator for the forward problem ...\n"; // fwdTimeStepSolver = Rythmos::timeStepNonlinearSolver<double>(); if (rythmosSolverPL->getEntryPtr("NonLinear Solver")) { const RCP<Teuchos::ParameterList> nonlinePL = sublist(rythmosSolverPL, "NonLinear Solver", true); fwdTimeStepSolver->setParameterList(nonlinePL); } // Force Default Integrator since this is needed for Observers rythmosPL->sublist("Integrator Settings").sublist("Integrator Selection"). set("Integrator Type","Default Integrator"); RCP<Rythmos::IntegratorBuilder<double> > ib = Rythmos::integratorBuilder<double>(); ib->setParameterList(rythmosPL); Thyra::ModelEvaluatorBase::InArgs<double> ic = thyraModel->getNominalValues(); RCP<Rythmos::IntegratorBase<double> > integrator = ib->create(thyraModel,ic,fwdTimeStepSolver); fwdStateIntegrator = Teuchos::rcp_dynamic_cast<Rythmos::DefaultIntegrator<double> >(integrator,true); fwdStateStepper = fwdStateIntegrator->getNonconstStepper(); if (Teuchos::nonnull(observer)) fwdStateIntegrator->setIntegrationObserver(observer); } else { TEUCHOS_TEST_FOR_EXCEPTION( appParams->isSublist("Rythmos") || appParams->isSublist("Rythmos Solver"), Teuchos::Exceptions::InvalidParameter, std::endl << "Error! Piro::RythmosSolver: must have either Rythmos or Rythmos Solver sublist "); } isInitialized = true; }
void Piro::RythmosSolver<Scalar>::initialize( const Teuchos::RCP<Teuchos::ParameterList> &appParams, const Teuchos::RCP< Thyra::ModelEvaluator<Scalar> > &in_model, const Teuchos::RCP<Rythmos::IntegrationObserverBase<Scalar> > &observer) { using Teuchos::ParameterList; using Teuchos::parameterList; using Teuchos::RCP; using Teuchos::rcp; // set some internals model = in_model; num_p = in_model->Np(); num_g = in_model->Ng(); // *out << "\nA) Get the base parameter list ...\n"; // RCP<Teuchos::ParameterList> rythmosPL = sublist(appParams, "Rythmos", true); rythmosPL->validateParameters(*getValidRythmosParameters(),0); { const std::string verbosity = rythmosPL->get("Verbosity Level", "VERB_DEFAULT"); if (verbosity == "VERB_NONE") solnVerbLevel = Teuchos::VERB_NONE; else if (verbosity == "VERB_DEFAULT") solnVerbLevel = Teuchos::VERB_DEFAULT; else if (verbosity == "VERB_LOW") solnVerbLevel = Teuchos::VERB_LOW; else if (verbosity == "VERB_MEDIUM") solnVerbLevel = Teuchos::VERB_MEDIUM; else if (verbosity == "VERB_HIGH") solnVerbLevel = Teuchos::VERB_HIGH; else if (verbosity == "VERB_EXTREME") solnVerbLevel = Teuchos::VERB_EXTREME; else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Unknown verbosity option specified in Piro_RythmosSolver."); } t_initial = rythmosPL->get("Initial Time", 0.0); t_final = rythmosPL->get("Final Time", 0.1); const std::string stepperType = rythmosPL->get("Stepper Type", "Backward Euler"); // *out << "\nC) Create and initalize the forward model ...\n"; // *out << "\nD) Create the stepper and integrator for the forward problem ...\n"; // if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "Rythmos") { Teuchos::RCP<Rythmos::TimeStepNonlinearSolver<Scalar> > rythmosTimeStepSolver = Rythmos::timeStepNonlinearSolver<Scalar>(); if (rythmosPL->getEntryPtr("NonLinear Solver")) { RCP<Teuchos::ParameterList> nonlinePL = sublist(rythmosPL, "NonLinear Solver", true); rythmosTimeStepSolver->setParameterList(nonlinePL); } fwdTimeStepSolver = rythmosTimeStepSolver; } else if (rythmosPL->get<std::string>("Nonlinear Solver Type") == "NOX") { #ifdef Piro_ENABLE_NOX Teuchos::RCP<Thyra::NOXNonlinearSolver> nox_solver = Teuchos::rcp(new Thyra::NOXNonlinearSolver); Teuchos::RCP<Teuchos::ParameterList> nox_params = Teuchos::rcp(new Teuchos::ParameterList); *nox_params = appParams->sublist("NOX"); nox_solver->setParameterList(nox_params); fwdTimeStepSolver = nox_solver; #else TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,"Requested NOX solver for a Rythmos Transient solve, Trilinos was not built with NOX enabled. Please rebuild Trilinos or use the native Rythmos nonlinear solver."); #endif } if (stepperType == "Backward Euler") { fwdStateStepper = Rythmos::backwardEulerStepper<Scalar> (model, fwdTimeStepSolver); fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true)); } else if (stepperType == "Forward Euler") { fwdStateStepper = Rythmos::forwardEulerStepper<Scalar> (model); fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true)); } else if (stepperType == "Explicit RK") { fwdStateStepper = Rythmos::explicitRKStepper<Scalar>(model); fwdStateStepper->setParameterList(sublist(rythmosPL, "Rythmos Stepper", true)); } else if (stepperType == "BDF") { Teuchos::RCP<Teuchos::ParameterList> BDFparams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true); Teuchos::RCP<Teuchos::ParameterList> BDFStepControlPL = Teuchos::sublist(BDFparams,"Step Control Settings"); fwdStateStepper = Teuchos::rcp( new Rythmos::ImplicitBDFStepper<Scalar>(model,fwdTimeStepSolver,BDFparams) ); fwdStateStepper->setInitialCondition(model->getNominalValues()); } else { // first (before failing) check to see if the user has added stepper factory typename std::map<std::string,Teuchos::RCP<RythmosStepperFactory<Scalar> > >::const_iterator stepFactItr = stepperFactories.find(stepperType); if(stepFactItr!=stepperFactories.end()) { // the user has added it, hot dog lets build a new stepper! Teuchos::RCP<Teuchos::ParameterList> stepperParams = Teuchos::sublist(rythmosPL, "Rythmos Stepper", true); // build the stepper using the factory fwdStateStepper = stepFactItr->second->buildStepper(model,fwdTimeStepSolver,stepperParams); } else { TEUCHOS_TEST_FOR_EXCEPTION( true, Teuchos::Exceptions::InvalidParameter, std::endl << "Error! Piro::Epetra::RythmosSolver: Invalid Steper Type: " << stepperType << std::endl); } } // Step control strategy { // If the stepper can accept a step control strategy, then attempt to build one. RCP<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> > scsa_stepper = Teuchos::rcp_dynamic_cast<Rythmos::StepControlStrategyAcceptingStepperBase<Scalar> >(fwdStateStepper); if (Teuchos::nonnull(scsa_stepper)) { const std::string step_control_strategy = rythmosPL->get("Step Control Strategy Type", "None"); if (step_control_strategy == "None") { // don't do anything, stepper will build default } else if (step_control_strategy == "ImplicitBDFRamping") { const RCP<Rythmos::ImplicitBDFStepperRampingStepControl<Scalar> > rscs = rcp(new Rythmos::ImplicitBDFStepperRampingStepControl<Scalar>); const RCP<ParameterList> p = parameterList(rythmosPL->sublist("Rythmos Step Control Strategy")); rscs->setParameterList(p); scsa_stepper->setStepControlStrategy(rscs); } else { TEUCHOS_TEST_FOR_EXCEPTION( true, std::logic_error, "Error! Piro::Epetra::RythmosSolver: Invalid step control strategy type: " << step_control_strategy << std::endl); } } } { const RCP<Teuchos::ParameterList> integrationControlPL = Teuchos::sublist(rythmosPL, "Rythmos Integration Control", true); RCP<Rythmos::DefaultIntegrator<Scalar> > defaultIntegrator; if (rythmosPL->get("Rythmos Integration Control Strategy", "Simple") == "Simple") { defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::simpleIntegrationControlStrategy<Scalar>(integrationControlPL)); } else if(rythmosPL->get<std::string>("Rythmos Integration Control Strategy") == "Ramping") { defaultIntegrator = Rythmos::controlledDefaultIntegrator<Scalar>(Rythmos::rampingIntegrationControlStrategy<Scalar>(integrationControlPL)); } fwdStateIntegrator = defaultIntegrator; } fwdStateIntegrator->setParameterList(sublist(rythmosPL, "Rythmos Integrator", true)); if (Teuchos::nonnull(observer)) { fwdStateIntegrator->setIntegrationObserver(observer); } isInitialized = true; }