int main(int argc, char *argv[]) {
int status=0; // 0 = pass, failures are incremented
bool success = true;
Teuchos::GlobalMPISession mpiSession(&argc,&argv);
using Teuchos::RCP;
using Teuchos::rcp;
RCP<Teuchos::FancyOStream> out(Teuchos::VerboseObjectBase::getDefaultOStream());
//***********************************************************
// Command-line argument for input file
//***********************************************************
std::string xmlfilename_coupled;
if(argc > 1){
if(!strcmp(argv[1],"--help")){
std::cout << "albany [inputfile.xml]" << std::endl;
std::exit(1);
}
else
xmlfilename_coupled = argv[1];
}
else
xmlfilename_coupled = "input.xml";
try {
RCP<Teuchos::Time> totalTime =
Teuchos::TimeMonitor::getNewTimer("AlbanySG: ***Total Time***");
RCP<Teuchos::Time> setupTime =
Teuchos::TimeMonitor::getNewTimer("AlbanySG: Setup Time");
Teuchos::TimeMonitor totalTimer(*totalTime); //start timer
//***********************************************************
// Set up coupled solver first to setup comm's
//***********************************************************
Teuchos::RCP<Epetra_Comm> globalComm =
Albany::createEpetraCommFromMpiComm(Albany_MPI_COMM_WORLD);
Albany::SolverFactory coupled_slvrfctry(xmlfilename_coupled,
Albany_MPI_COMM_WORLD);
Teuchos::ParameterList& coupledParams = coupled_slvrfctry.getParameters();
Teuchos::ParameterList& coupledSystemParams =
coupledParams.sublist("Coupled System");
Teuchos::Array<std::string> model_filenames =
coupledSystemParams.get<Teuchos::Array<std::string> >("Model XML Files");
int num_models = model_filenames.size();
Teuchos::Array< RCP<Albany::Application> > apps(num_models);
Teuchos::Array< RCP<EpetraExt::ModelEvaluator> > models(num_models);
Teuchos::Array< RCP<Teuchos::ParameterList> > piroParams(num_models);
Teuchos::RCP< Teuchos::ParameterList> coupledPiroParams =
Teuchos::rcp(&(coupledParams.sublist("Piro")),false);
Teuchos::RCP<Piro::Epetra::StokhosSolver> coupledSolver =
Teuchos::rcp(new Piro::Epetra::StokhosSolver(coupledPiroParams,
globalComm));
Teuchos::RCP<const Epetra_Comm> app_comm = coupledSolver->getSpatialComm();
// Set up each model
Teuchos::Array< Teuchos::RCP<NOX::Epetra::Observer> > observers(num_models);
for (int m=0; m<num_models; m++) {
Albany::SolverFactory slvrfctry(
model_filenames[m],
Albany::getMpiCommFromEpetraComm(*app_comm));
models[m] = slvrfctry.createAlbanyAppAndModel(apps[m], app_comm);
Teuchos::ParameterList& appParams = slvrfctry.getParameters();
piroParams[m] = Teuchos::rcp(&(appParams.sublist("Piro")),false);
observers[m] = Teuchos::rcp(new Albany_NOXObserver(apps[m]));
}
// Setup network model
std::string network_name =
coupledSystemParams.get("Network Model", "Param To Response");
RCP<Piro::Epetra::AbstractNetworkModel> network_model;
if (network_name == "Param To Response")
network_model = rcp(new Piro::Epetra::ParamToResponseNetworkModel);
else if (network_name == "Reactor Network")
network_model = rcp(new Albany::ReactorNetworkModel(1));
else
TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error, "Invalid network model name " << network_name);
RCP<EpetraExt::ModelEvaluator> coupledModel =
rcp(new Piro::Epetra::NECoupledModelEvaluator(models, piroParams,
network_model,
coupledPiroParams,
globalComm,
observers));
coupledSolver->setup(coupledModel);
// Solve coupled system
EpetraExt::ModelEvaluator::InArgs inArgs = coupledSolver->createInArgs();
EpetraExt::ModelEvaluator::OutArgs outArgs = coupledSolver->createOutArgs();
for (int i=0; i<inArgs.Np(); i++)
if (inArgs.supports(EpetraExt::ModelEvaluator::IN_ARG_p_sg, i))
inArgs.set_p_sg(i, coupledSolver->get_p_sg_init(i));
for (int i=0; i<outArgs.Ng(); i++)
if (outArgs.supports(EpetraExt::ModelEvaluator::OUT_ARG_g_sg, i)) {
RCP<Stokhos::EpetraVectorOrthogPoly> g_sg =
coupledSolver->create_g_sg(i);
outArgs.set_g_sg(i, g_sg);
}
coupledSolver->evalModel(inArgs, outArgs);
// Print results
bool printResponse =
coupledSystemParams.get("Print Response Expansion", true);
int idx = outArgs.Ng()-1;
Teuchos::RCP<Stokhos::EpetraVectorOrthogPoly> g_sg =
outArgs.get_g_sg(idx);
Teuchos::RCP<Stokhos::SGModelEvaluator> sg_model =
coupledSolver->get_sg_model();
Teuchos::RCP<Stokhos::EpetraVectorOrthogPoly> g_sg_local =
//sg_model->import_solution_poly(*(g_sg->getBlockVector()));
g_sg;
Epetra_Vector g_mean(*(g_sg->coefficientMap()));
Epetra_Vector g_std_dev(*(g_sg->coefficientMap()));
g_sg->computeMean(g_mean);
g_sg->computeStandardDeviation(g_std_dev);
RCP<Epetra_Vector> g_mean_local = rcp(&g_mean,false);
RCP<Epetra_Vector> g_std_dev_local = rcp(&g_std_dev,false);
if (g_mean.Map().DistributedGlobal()) {
Epetra_LocalMap local_map(g_mean.GlobalLength(), 0,
g_mean.Map().Comm());
g_mean_local = rcp(new Epetra_Vector(local_map));
g_std_dev_local = rcp(new Epetra_Vector(local_map));
Epetra_Import importer(local_map, g_mean.Map());
g_mean_local->Import(g_mean, importer, Insert);
g_std_dev_local->Import(g_std_dev, importer, Insert);
}
out->precision(16);
*out << std::endl
<< "Final value of coupling variables:" << std::endl
<< "Mean:" << std::endl << *g_mean_local << std::endl
<< "Std. Dev.:" << std::endl << *g_std_dev_local << std::endl;
if (printResponse)
*out << "PCE:" << std::endl << *g_sg_local << std::endl;
status += coupled_slvrfctry.checkSGTestResults(
0,
g_sg_local,
g_mean_local.get(),
g_std_dev_local.get());
*out << "\nNumber of Failed Comparisons: " << status << std::endl;
}
TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
if (!success) status+=10000;
Teuchos::TimeMonitor::summarize(*out,false,true,false/*zero timers*/);
return status;
}
EpetraExt::MultiPointModelEvaluator::MultiPointModelEvaluator(
Teuchos::RefCountPtr<EpetraExt::ModelEvaluator> underlyingME_,
const Teuchos::RefCountPtr<EpetraExt::MultiComm> &globalComm_,
const std::vector<Epetra_Vector*> initGuessVec_,
Teuchos::RefCountPtr<std::vector< Teuchos::RefCountPtr<Epetra_Vector> > > q_vec_,
Teuchos::RefCountPtr<std::vector< Teuchos::RefCountPtr<Epetra_Vector> > > matching_vec_
) :
underlyingME(underlyingME_),
globalComm(globalComm_),
q_vec(q_vec_),
underlyingNg(0),
timeStepsOnTimeDomain(globalComm_->NumTimeStepsOnDomain()),
numTimeDomains(globalComm_->NumSubDomains()),
timeDomain(globalComm_->SubDomainRank()),
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
rowStencil_int(0),
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
rowStencil_LL(0),
#endif
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
rowIndex_int(0),
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
rowIndex_LL(0),
#endif
matching_vec(matching_vec_)
{
using Teuchos::as;
if (globalComm->MyPID()==0) {
std::cout << "----------MultiPoint Partition Info------------"
<< "\n\tNumProcs = " << globalComm->NumProc()
<< "\n\tSpatial Decomposition = " << globalComm->SubDomainComm().NumProc()
<< "\n\tNumber of Domains = " << numTimeDomains
<< "\n\tSteps on Domain 0 = " << timeStepsOnTimeDomain
<< "\n\tTotal Number of Steps = " << globalComm->NumTimeSteps();
std::cout << "\n-----------------------------------------------" << std::endl;
}
// Construct global block matrix graph from split W and stencil,
// which is just diagonal in this case
split_W = Teuchos::rcp_dynamic_cast<Epetra_RowMatrix>(underlyingME->create_W());
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
if(split_W->RowMatrixRowMap().GlobalIndicesInt()) {
longlong = false;
rowStencil_int = new std::vector< std::vector<int> >(timeStepsOnTimeDomain);
rowIndex_int = new std::vector<int>;
for (int i=0; i < timeStepsOnTimeDomain; i++) {
(*rowStencil_int)[i].push_back(0);
(*rowIndex_int).push_back(i + globalComm->FirstTimeStepOnDomain());
}
block_W = Teuchos::rcp(new EpetraExt::BlockCrsMatrix(*split_W,
*rowStencil_int, *rowIndex_int, *globalComm));
}
else
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
if(split_W->RowMatrixRowMap().GlobalIndicesInt()) {
longlong = true;
rowStencil_LL = new std::vector< std::vector<long long> >(timeStepsOnTimeDomain);
rowIndex_LL = new std::vector<long long>;
for (int i=0; i < timeStepsOnTimeDomain; i++) {
(*rowStencil_LL)[i].push_back(0);
(*rowIndex_LL).push_back(i + globalComm->FirstTimeStepOnDomain());
}
block_W = Teuchos::rcp(new EpetraExt::BlockCrsMatrix(*split_W,
*rowStencil_LL, *rowIndex_LL, *globalComm));
}
else
#endif
throw "EpetraExt::MultiPointModelEvaluator::MultiPointModelEvaluator: Global indices unknown";
// Test for g vector
EpetraExt::ModelEvaluator::OutArgs underlyingOutArgs = underlyingME->createOutArgs();
underlyingNg = underlyingOutArgs.Ng();
if (underlyingNg) {
if (underlyingOutArgs.supports(OUT_ARG_DgDp,0,0).supports(DERIV_TRANS_MV_BY_ROW))
orientation_DgDp = DERIV_TRANS_MV_BY_ROW;
else
orientation_DgDp = DERIV_MV_BY_COL;
}
// This code assumes 2 parameter vectors, 1 for opt, second for MultiPoint states
TEUCHOS_TEST_FOR_EXCEPT(underlyingOutArgs.Np()!=2);
// temporary quantities
const Epetra_Map& split_map = split_W->RowMatrixRowMap();
num_p0 = underlyingME_->get_p_map(0)->NumMyElements();
if (underlyingNg) num_g0 = underlyingME_->get_g_map(0)->NumMyElements();
else num_g0 = 0;
num_dg0dp0 = num_g0 * num_p0;
// Construct global solution vector, residual vector -- local storage
block_x = new EpetraExt::BlockVector(split_map, block_W->RowMap());
block_f = new EpetraExt::BlockVector(*block_x);
block_DfDp = new EpetraExt::BlockMultiVector(split_map, block_W->RowMap(), num_p0);
if (underlyingNg)
block_DgDx = new EpetraExt::BlockMultiVector(split_map, block_W->RowMap(), num_g0);
// Allocate local storage of epetra vectors
split_x = Teuchos::rcp(new Epetra_Vector(split_map));
split_f = Teuchos::rcp(new Epetra_Vector(split_map));
split_DfDp = Teuchos::rcp(new Epetra_MultiVector(split_map, num_p0));
if (underlyingNg)
split_DgDx = Teuchos::rcp(new Epetra_MultiVector(split_map, num_g0));
if (underlyingNg) {
if(orientation_DgDp == DERIV_TRANS_MV_BY_ROW)
split_DgDp = Teuchos::rcp(new Epetra_MultiVector(*(underlyingME_->get_p_map(0)), num_g0));
else
split_DgDp = Teuchos::rcp(new Epetra_MultiVector(*(underlyingME_->get_g_map(0)), num_p0));
}
if (underlyingNg)
split_g = Teuchos::rcp(new Epetra_Vector(*(underlyingME_->get_g_map(0))));
// Packaging required for getting multivectors back as Derivatives
derivMV_DfDp = new EpetraExt::ModelEvaluator::DerivativeMultiVector(split_DfDp);
deriv_DfDp = new EpetraExt::ModelEvaluator::Derivative(*derivMV_DfDp);
if (underlyingNg) {
derivMV_DgDx = new EpetraExt::ModelEvaluator::DerivativeMultiVector(split_DgDx, DERIV_TRANS_MV_BY_ROW);
deriv_DgDx = new EpetraExt::ModelEvaluator::Derivative(*derivMV_DgDx);
derivMV_DgDp = new EpetraExt::ModelEvaluator::DerivativeMultiVector(split_DgDp, orientation_DgDp);
deriv_DgDp = new EpetraExt::ModelEvaluator::Derivative(*derivMV_DgDp);
}
// For 4D, we will need the overlap vector and importer between them
// Overlap not needed for MultiPoint -- no overlap between blocks
/* solutionOverlap = new EpetraExt::BlockVector(split_W->RowMatrixRowMap(),
block_W->ColMap());
overlapImporter = new Epetra_Import(solutionOverlap->Map(), block_x->Map());
*/
// Load initial guess into block solution vector
solution_init = Teuchos::rcp(new EpetraExt::BlockVector(*block_x));
if(longlong) {
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
for (int i=0; i < timeStepsOnTimeDomain; i++)
solution_init->LoadBlockValues(*(initGuessVec_[i]), (*rowIndex_LL)[i]);
#endif
}
else {
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
for (int i=0; i < timeStepsOnTimeDomain; i++)
solution_init->LoadBlockValues(*(initGuessVec_[i]), (*rowIndex_int)[i]);
#endif
}
//Prepare logic for matching problem
if (Teuchos::is_null(matching_vec)) matchingProblem = false;
else matchingProblem = true;
if (matchingProblem) {
TEUCHOS_TEST_FOR_EXCEPT(as<int>(matching_vec->size())!=timeStepsOnTimeDomain);
TEUCHOS_TEST_FOR_EXCEPT(!(*matching_vec)[0]->Map().SameAs(*(underlyingME_->get_g_map(0))));
TEUCHOS_TEST_FOR_EXCEPT(num_g0 != 1); //This restriction may be lifted later
}
}
Piro::Epetra::NOXSolver::NOXSolver(
Teuchos::RCP<Teuchos::ParameterList> piroParams_,
Teuchos::RCP<EpetraExt::ModelEvaluator> model_,
Teuchos::RCP<NOX::Epetra::Observer> observer_,
Teuchos::RCP<NOX::Epetra::ModelEvaluatorInterface> custom_interface,
Teuchos::RCP<NOX::Epetra::LinearSystem> custom_linsys
) :
piroParams(piroParams_),
model(model_),
observer(observer_),
utils(piroParams->sublist("NOX").sublist("Printing"))
{
Teuchos::RCP<Teuchos::ParameterList> noxParams =
Teuchos::rcp(&(piroParams->sublist("NOX")),false);
Teuchos::ParameterList& printParams = noxParams->sublist("Printing");
std::string jacobianSource = piroParams->get("Jacobian Operator", "Have Jacobian");
bool leanMatrixFree = piroParams->get("Lean Matrix Free",false);
Teuchos::ParameterList& noxstratlsParams = noxParams->
sublist("Direction").sublist("Newton").sublist("Stratimikos Linear Solver");
// Inexact Newton must be set in a second sublist when using
// Stratimikos: This code snippet sets it automatically
bool inexact = (noxParams->sublist("Direction").sublist("Newton").
get("Forcing Term Method", "Constant") != "Constant");
if (inexact)
noxstratlsParams.sublist("NOX Stratimikos Options").
set("Use Linear Solve Tolerance From NOX", inexact);
if (jacobianSource == "Matrix-Free") {
if (piroParams->isParameter("Matrix-Free Perturbation")) {
model = Teuchos::rcp(new Piro::Epetra::MatrixFreeDecorator(model,
piroParams->get<double>("Matrix-Free Perturbation")));
}
else model = Teuchos::rcp(new Piro::Epetra::MatrixFreeDecorator(model));
}
// Grab some modelEval stuff from underlying model
EpetraExt::ModelEvaluator::InArgs inargs = model->createInArgs();
num_p = inargs.Np();
EpetraExt::ModelEvaluator::OutArgs outargs = model->createOutArgs();
num_g = outargs.Ng();
// Create initial guess
Teuchos::RCP<const Epetra_Vector> u = model->get_x_init();
currentSolution = Teuchos::rcp(new NOX::Epetra::Vector(*u));
// Create NOX interface from model evaluator
if (custom_interface != Teuchos::null)
interface = custom_interface;
else
interface = Teuchos::rcp(new NOX::Epetra::ModelEvaluatorInterface(model));
Teuchos::RCP<NOX::Epetra::Interface::Required> iReq = interface;
// Create the Jacobian matrix (unless flag is set to do it numerically)
Teuchos::RCP<Epetra_Operator> A;
Teuchos::RCP<NOX::Epetra::Interface::Jacobian> iJac;
if (jacobianSource == "Have Jacobian" || jacobianSource == "Matrix-Free") {
A = model->create_W();
iJac = interface;
}
else if (jacobianSource == "Finite Difference") {
A = Teuchos::rcp(new NOX::Epetra::FiniteDifference(printParams,
iReq, *currentSolution));
iJac = Teuchos::rcp_dynamic_cast<NOX::Epetra::Interface::Jacobian>(A);
}
else
TEUCHOS_TEST_FOR_EXCEPTION(true, Teuchos::Exceptions::InvalidParameter,
"Error in Piro::Epetra::NOXSolver " <<
"Invalid value for parameter \" Jacobian Operator\"= " <<
jacobianSource << std::endl);
// Create separate preconditioner if the model supports it
/* NOTE: How do we want to decide between using an
* available preconditioner: (1) If the model supports
* it, then we use it, or (2) if a parameter list says
* User_Defined ? [Below, logic is ooption (1).]
*/
Teuchos::RCP<EpetraExt::ModelEvaluator::Preconditioner> WPrec;
if (outargs.supports(EpetraExt::ModelEvaluator::OUT_ARG_WPrec))
WPrec = model->create_WPrec();
// Create the linear system
if (custom_linsys != Teuchos::null)
linsys = custom_linsys;
else {
if (WPrec != Teuchos::null) {
Teuchos::RCP<NOX::Epetra::Interface::Preconditioner> iPrec = interface;
linsys = Teuchos::rcp(new NOX::Epetra::LinearSystemStratimikos(
printParams,
noxstratlsParams, iJac, A, iPrec, WPrec->PrecOp,
*currentSolution, WPrec->isAlreadyInverted));
}
else {
linsys = Teuchos::rcp(new NOX::Epetra::LinearSystemStratimikos(
printParams,
noxstratlsParams, iJac, A, *currentSolution));
}
}
// Build NOX group
grp = Teuchos::rcp(new NOX::Epetra::Group(printParams, iReq,
*currentSolution, linsys));
// Saves one resid calculation per solve, but not as safe
if (leanMatrixFree) grp->disableLinearResidualComputation(true);
// Create the Solver convergence test
Teuchos::ParameterList& statusParams = noxParams->sublist("Status Tests");
Teuchos::RCP<NOX::StatusTest::Generic> statusTests =
NOX::StatusTest::buildStatusTests(statusParams, utils);
// Create the solver
solver = NOX::Solver::buildSolver(grp, statusTests, noxParams);
// Create transpose linear solver
Teuchos::RCP<LOCA::Abstract::Factory> epetraFactory =
Teuchos::rcp(new LOCA::Epetra::Factory);
globalData = LOCA::createGlobalData(piroParams, epetraFactory);
LOCA::Epetra::TransposeLinearSystem::Factory tls_factory(globalData);
tls_strategy = tls_factory.create(piroParams, linsys);
}