Exemple #1
0
int main(int argc, char *argv[]) {

  int status=0; // 0 = pass, failures are incremented
  bool success = true;
  Teuchos::GlobalMPISession mpiSession(&argc,&argv);

  using Teuchos::RCP;
  using Teuchos::rcp;

  RCP<Teuchos::FancyOStream> out(Teuchos::VerboseObjectBase::getDefaultOStream());
  
  //***********************************************************
  // Command-line argument for input file
  //***********************************************************

  std::string xmlfilename_coupled;
  if(argc > 1){
    if(!strcmp(argv[1],"--help")){
      std::cout << "albany [inputfile.xml]" << std::endl;
      std::exit(1);
    }
    else
      xmlfilename_coupled = argv[1];
  }
  else
    xmlfilename_coupled = "input.xml";


  try {

    RCP<Teuchos::Time> totalTime = 
      Teuchos::TimeMonitor::getNewTimer("AlbanySG: ***Total Time***");
    RCP<Teuchos::Time> setupTime = 
      Teuchos::TimeMonitor::getNewTimer("AlbanySG: Setup Time");
    Teuchos::TimeMonitor totalTimer(*totalTime); //start timer

    //***********************************************************
    // Set up coupled solver first to setup comm's
    //***********************************************************
    Teuchos::RCP<Epetra_Comm> globalComm = 
      Albany::createEpetraCommFromMpiComm(Albany_MPI_COMM_WORLD);
    Albany::SolverFactory coupled_slvrfctry(xmlfilename_coupled, 
					    Albany_MPI_COMM_WORLD);
    Teuchos::ParameterList& coupledParams = coupled_slvrfctry.getParameters();
    Teuchos::ParameterList& coupledSystemParams = 
      coupledParams.sublist("Coupled System");
    Teuchos::Array<std::string> model_filenames =
      coupledSystemParams.get<Teuchos::Array<std::string> >("Model XML Files");
    int num_models = model_filenames.size();
    Teuchos::Array< RCP<Albany::Application> > apps(num_models);
    Teuchos::Array< RCP<EpetraExt::ModelEvaluator> > models(num_models);
    Teuchos::Array< RCP<Teuchos::ParameterList> > piroParams(num_models);
    Teuchos::RCP< Teuchos::ParameterList> coupledPiroParams = 
      Teuchos::rcp(&(coupledParams.sublist("Piro")),false);
    Teuchos::RCP<Piro::Epetra::StokhosSolver> coupledSolver =
      Teuchos::rcp(new Piro::Epetra::StokhosSolver(coupledPiroParams, 
						   globalComm));
    Teuchos::RCP<const Epetra_Comm> app_comm = coupledSolver->getSpatialComm();

    // Set up each model
    Teuchos::Array< Teuchos::RCP<NOX::Epetra::Observer> > observers(num_models);
    for (int m=0; m<num_models; m++) {
      Albany::SolverFactory slvrfctry(
	model_filenames[m], 
	Albany::getMpiCommFromEpetraComm(*app_comm));
      models[m] = slvrfctry.createAlbanyAppAndModel(apps[m], app_comm);
      Teuchos::ParameterList& appParams = slvrfctry.getParameters();
      piroParams[m] = Teuchos::rcp(&(appParams.sublist("Piro")),false);
      observers[m] = Teuchos::rcp(new Albany_NOXObserver(apps[m]));
    }

    // Setup network model
    std::string network_name = 
      coupledSystemParams.get("Network Model", "Param To Response");
    RCP<Piro::Epetra::AbstractNetworkModel> network_model;
    if (network_name == "Param To Response")
      network_model = rcp(new Piro::Epetra::ParamToResponseNetworkModel);
    else if (network_name == "Reactor Network")
      network_model = rcp(new Albany::ReactorNetworkModel(1));
    else
      TEUCHOS_TEST_FOR_EXCEPTION(
	true, std::logic_error, "Invalid network model name " << network_name);
    RCP<EpetraExt::ModelEvaluator> coupledModel =
      rcp(new Piro::Epetra::NECoupledModelEvaluator(models, piroParams,
						    network_model,
						    coupledPiroParams, 
						    globalComm,
						    observers));
    coupledSolver->setup(coupledModel);

    // Solve coupled system
    EpetraExt::ModelEvaluator::InArgs inArgs = coupledSolver->createInArgs();
    EpetraExt::ModelEvaluator::OutArgs outArgs = coupledSolver->createOutArgs();
    for (int i=0; i<inArgs.Np(); i++)
      if (inArgs.supports(EpetraExt::ModelEvaluator::IN_ARG_p_sg, i))
	inArgs.set_p_sg(i, coupledSolver->get_p_sg_init(i));
    for (int i=0; i<outArgs.Ng(); i++) 
      if (outArgs.supports(EpetraExt::ModelEvaluator::OUT_ARG_g_sg, i)) {
	RCP<Stokhos::EpetraVectorOrthogPoly> g_sg = 
	  coupledSolver->create_g_sg(i);
	outArgs.set_g_sg(i, g_sg);
      }
    coupledSolver->evalModel(inArgs, outArgs);

    // Print results
    bool printResponse = 
      coupledSystemParams.get("Print Response Expansion", true);
    int idx = outArgs.Ng()-1;
    Teuchos::RCP<Stokhos::EpetraVectorOrthogPoly> g_sg = 
      outArgs.get_g_sg(idx);
    Teuchos::RCP<Stokhos::SGModelEvaluator> sg_model =
      coupledSolver->get_sg_model();
    Teuchos::RCP<Stokhos::EpetraVectorOrthogPoly> g_sg_local = 
      //sg_model->import_solution_poly(*(g_sg->getBlockVector()));
      g_sg;
    Epetra_Vector g_mean(*(g_sg->coefficientMap()));
    Epetra_Vector g_std_dev(*(g_sg->coefficientMap()));
    g_sg->computeMean(g_mean);
    g_sg->computeStandardDeviation(g_std_dev);
    RCP<Epetra_Vector> g_mean_local = rcp(&g_mean,false);
    RCP<Epetra_Vector> g_std_dev_local = rcp(&g_std_dev,false);
    if (g_mean.Map().DistributedGlobal()) {
      Epetra_LocalMap local_map(g_mean.GlobalLength(), 0, 
				g_mean.Map().Comm());
      g_mean_local = rcp(new Epetra_Vector(local_map));
      g_std_dev_local = rcp(new Epetra_Vector(local_map));
      Epetra_Import importer(local_map, g_mean.Map());
      g_mean_local->Import(g_mean, importer, Insert);
      g_std_dev_local->Import(g_std_dev, importer, Insert);
    }
    out->precision(16);
    *out << std::endl
	 << "Final value of coupling variables:" << std::endl
	 << "Mean:" << std::endl << *g_mean_local << std::endl
	 << "Std. Dev.:" << std::endl << *g_std_dev_local << std::endl;
    if (printResponse)
      *out << "PCE:" << std::endl << *g_sg_local << std::endl;

    status += coupled_slvrfctry.checkSGTestResults(
        0,
        g_sg_local,
        g_mean_local.get(),
        g_std_dev_local.get());
    *out << "\nNumber of Failed Comparisons: " << status << std::endl;
  }

  TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
  if (!success) status+=10000;

  Teuchos::TimeMonitor::summarize(*out,false,true,false/*zero timers*/);
  return status;
}
EpetraExt::MultiPointModelEvaluator::MultiPointModelEvaluator(
    Teuchos::RefCountPtr<EpetraExt::ModelEvaluator> underlyingME_,
    const Teuchos::RefCountPtr<EpetraExt::MultiComm> &globalComm_,
    const std::vector<Epetra_Vector*> initGuessVec_,
    Teuchos::RefCountPtr<std::vector< Teuchos::RefCountPtr<Epetra_Vector> > >  q_vec_,
    Teuchos::RefCountPtr<std::vector< Teuchos::RefCountPtr<Epetra_Vector> > >  matching_vec_
    ) :
    underlyingME(underlyingME_),
    globalComm(globalComm_),
    q_vec(q_vec_),
    underlyingNg(0),
    timeStepsOnTimeDomain(globalComm_->NumTimeStepsOnDomain()),
    numTimeDomains(globalComm_->NumSubDomains()),
    timeDomain(globalComm_->SubDomainRank()),
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
    rowStencil_int(0),
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
    rowStencil_LL(0),
#endif
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
    rowIndex_int(0),
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
    rowIndex_LL(0),
#endif
    matching_vec(matching_vec_)
{
  using Teuchos::as;
  if (globalComm->MyPID()==0) {
     std::cout  << "----------MultiPoint Partition Info------------"
           << "\n\tNumProcs              = " << globalComm->NumProc()
           << "\n\tSpatial Decomposition = " << globalComm->SubDomainComm().NumProc()
           << "\n\tNumber of Domains     = " << numTimeDomains
           << "\n\tSteps on Domain 0     = " << timeStepsOnTimeDomain
           << "\n\tTotal Number of Steps = " << globalComm->NumTimeSteps();
    std::cout   << "\n-----------------------------------------------" << std::endl;
    }

   // Construct global block matrix graph from split W and stencil,
   // which is just diagonal in this case

   split_W = Teuchos::rcp_dynamic_cast<Epetra_RowMatrix>(underlyingME->create_W());

#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
   if(split_W->RowMatrixRowMap().GlobalIndicesInt()) {
     longlong = false;
     rowStencil_int = new std::vector< std::vector<int> >(timeStepsOnTimeDomain);
     rowIndex_int = new std::vector<int>;
     for (int i=0; i < timeStepsOnTimeDomain; i++) {
       (*rowStencil_int)[i].push_back(0);
       (*rowIndex_int).push_back(i + globalComm->FirstTimeStepOnDomain());
     }
     block_W = Teuchos::rcp(new EpetraExt::BlockCrsMatrix(*split_W,
                               *rowStencil_int, *rowIndex_int, *globalComm));
   }
   else
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
   if(split_W->RowMatrixRowMap().GlobalIndicesInt()) {
     longlong = true;
     rowStencil_LL = new std::vector< std::vector<long long> >(timeStepsOnTimeDomain);
     rowIndex_LL = new std::vector<long long>;
     for (int i=0; i < timeStepsOnTimeDomain; i++) {
       (*rowStencil_LL)[i].push_back(0);
       (*rowIndex_LL).push_back(i + globalComm->FirstTimeStepOnDomain());
     }
     block_W = Teuchos::rcp(new EpetraExt::BlockCrsMatrix(*split_W,
                               *rowStencil_LL, *rowIndex_LL, *globalComm));
   }
   else
#endif
     throw "EpetraExt::MultiPointModelEvaluator::MultiPointModelEvaluator: Global indices unknown";

   // Test for g vector
   EpetraExt::ModelEvaluator::OutArgs underlyingOutArgs = underlyingME->createOutArgs();

   underlyingNg = underlyingOutArgs.Ng();
   if (underlyingNg) {
     if (underlyingOutArgs.supports(OUT_ARG_DgDp,0,0).supports(DERIV_TRANS_MV_BY_ROW))
       orientation_DgDp = DERIV_TRANS_MV_BY_ROW;
     else
       orientation_DgDp = DERIV_MV_BY_COL;
   }

   // This code assumes 2 parameter vectors, 1 for opt, second for MultiPoint states
   TEUCHOS_TEST_FOR_EXCEPT(underlyingOutArgs.Np()!=2);

   // temporary quantities
   const Epetra_Map& split_map = split_W->RowMatrixRowMap();
   num_p0 =  underlyingME_->get_p_map(0)->NumMyElements();
   if (underlyingNg)  num_g0 = underlyingME_->get_g_map(0)->NumMyElements();
   else num_g0 = 0;
   num_dg0dp0 = num_g0 * num_p0;

   // Construct global solution vector, residual vector -- local storage
   block_x = new EpetraExt::BlockVector(split_map, block_W->RowMap());
   block_f = new EpetraExt::BlockVector(*block_x);
   block_DfDp = new EpetraExt::BlockMultiVector(split_map, block_W->RowMap(), num_p0);
    if (underlyingNg)
   block_DgDx = new EpetraExt::BlockMultiVector(split_map, block_W->RowMap(), num_g0);

   // Allocate local storage of epetra vectors
   split_x = Teuchos::rcp(new Epetra_Vector(split_map));
   split_f = Teuchos::rcp(new Epetra_Vector(split_map));
   split_DfDp = Teuchos::rcp(new Epetra_MultiVector(split_map, num_p0));
   if (underlyingNg)
     split_DgDx = Teuchos::rcp(new Epetra_MultiVector(split_map, num_g0));
   if (underlyingNg) {
     if(orientation_DgDp == DERIV_TRANS_MV_BY_ROW)
       split_DgDp = Teuchos::rcp(new Epetra_MultiVector(*(underlyingME_->get_p_map(0)), num_g0));
     else
       split_DgDp = Teuchos::rcp(new Epetra_MultiVector(*(underlyingME_->get_g_map(0)), num_p0));
   }
   if (underlyingNg)
     split_g = Teuchos::rcp(new Epetra_Vector(*(underlyingME_->get_g_map(0))));

   // Packaging required for getting multivectors back as Derivatives
   derivMV_DfDp = new EpetraExt::ModelEvaluator::DerivativeMultiVector(split_DfDp);
   deriv_DfDp = new EpetraExt::ModelEvaluator::Derivative(*derivMV_DfDp);
   if (underlyingNg)  {
     derivMV_DgDx = new EpetraExt::ModelEvaluator::DerivativeMultiVector(split_DgDx, DERIV_TRANS_MV_BY_ROW);
     deriv_DgDx = new EpetraExt::ModelEvaluator::Derivative(*derivMV_DgDx);
     derivMV_DgDp = new EpetraExt::ModelEvaluator::DerivativeMultiVector(split_DgDp, orientation_DgDp);
     deriv_DgDp = new EpetraExt::ModelEvaluator::Derivative(*derivMV_DgDp);
   }

   // For 4D, we will need the overlap vector and importer between them
   // Overlap not needed for MultiPoint -- no overlap between blocks
   /*   solutionOverlap = new EpetraExt::BlockVector(split_W->RowMatrixRowMap(),
                                                     block_W->ColMap());
        overlapImporter = new Epetra_Import(solutionOverlap->Map(), block_x->Map());
   */

   // Load initial guess into block solution vector
   solution_init = Teuchos::rcp(new EpetraExt::BlockVector(*block_x));

   if(longlong) {
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
     for (int i=0; i < timeStepsOnTimeDomain; i++)
             solution_init->LoadBlockValues(*(initGuessVec_[i]), (*rowIndex_LL)[i]);
#endif
   }
   else {
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
     for (int i=0; i < timeStepsOnTimeDomain; i++)
             solution_init->LoadBlockValues(*(initGuessVec_[i]), (*rowIndex_int)[i]);
#endif
   }


   //Prepare logic for matching problem
   if (Teuchos::is_null(matching_vec))  matchingProblem = false;
   else matchingProblem = true;

   if (matchingProblem) {
     TEUCHOS_TEST_FOR_EXCEPT(as<int>(matching_vec->size())!=timeStepsOnTimeDomain);
     TEUCHOS_TEST_FOR_EXCEPT(!(*matching_vec)[0]->Map().SameAs(*(underlyingME_->get_g_map(0))));
     TEUCHOS_TEST_FOR_EXCEPT(num_g0 != 1); //This restriction may be lifted later
   }
}
Piro::Epetra::NOXSolver::NOXSolver(
  Teuchos::RCP<Teuchos::ParameterList> piroParams_,
  Teuchos::RCP<EpetraExt::ModelEvaluator> model_,
  Teuchos::RCP<NOX::Epetra::Observer> observer_,
  Teuchos::RCP<NOX::Epetra::ModelEvaluatorInterface> custom_interface,
  Teuchos::RCP<NOX::Epetra::LinearSystem> custom_linsys
) :
  piroParams(piroParams_),
  model(model_),
  observer(observer_),
  utils(piroParams->sublist("NOX").sublist("Printing"))
{
  Teuchos::RCP<Teuchos::ParameterList> noxParams =
	Teuchos::rcp(&(piroParams->sublist("NOX")),false);
  Teuchos::ParameterList& printParams = noxParams->sublist("Printing");

  std::string jacobianSource = piroParams->get("Jacobian Operator", "Have Jacobian");
  bool leanMatrixFree = piroParams->get("Lean Matrix Free",false);

  Teuchos::ParameterList& noxstratlsParams = noxParams->
        sublist("Direction").sublist("Newton").sublist("Stratimikos Linear Solver");

  // Inexact Newton must be set in a second sublist when using 
  // Stratimikos: This code snippet sets it automatically
  bool inexact = (noxParams->sublist("Direction").sublist("Newton").
                  get("Forcing Term Method", "Constant") != "Constant");
  if (inexact)
    noxstratlsParams.sublist("NOX Stratimikos Options").
       set("Use Linear Solve Tolerance From NOX", inexact);


  if (jacobianSource == "Matrix-Free") {
    if (piroParams->isParameter("Matrix-Free Perturbation")) {
      model = Teuchos::rcp(new Piro::Epetra::MatrixFreeDecorator(model,
                           piroParams->get<double>("Matrix-Free Perturbation")));
    }
    else model = Teuchos::rcp(new Piro::Epetra::MatrixFreeDecorator(model));
  }

  // Grab some modelEval stuff from underlying model
  EpetraExt::ModelEvaluator::InArgs inargs = model->createInArgs();
  num_p = inargs.Np();
  EpetraExt::ModelEvaluator::OutArgs outargs = model->createOutArgs();
  num_g = outargs.Ng();

  // Create initial guess
  Teuchos::RCP<const Epetra_Vector> u = model->get_x_init();
  currentSolution = Teuchos::rcp(new NOX::Epetra::Vector(*u));

  // Create NOX interface from model evaluator
  if (custom_interface != Teuchos::null)
    interface = custom_interface;
  else
    interface = Teuchos::rcp(new NOX::Epetra::ModelEvaluatorInterface(model));
  Teuchos::RCP<NOX::Epetra::Interface::Required> iReq = interface;

  // Create the Jacobian matrix (unless flag is set to do it numerically)
  Teuchos::RCP<Epetra_Operator> A;
  Teuchos::RCP<NOX::Epetra::Interface::Jacobian> iJac;

  if (jacobianSource == "Have Jacobian" || jacobianSource == "Matrix-Free") {
    A = model->create_W();
    iJac = interface;
  }
  else if (jacobianSource == "Finite Difference") {
    A = Teuchos::rcp(new NOX::Epetra::FiniteDifference(printParams,
                                            iReq, *currentSolution));
    iJac = Teuchos::rcp_dynamic_cast<NOX::Epetra::Interface::Jacobian>(A);
  }
  else
    TEUCHOS_TEST_FOR_EXCEPTION(true, Teuchos::Exceptions::InvalidParameter,
                 "Error in Piro::Epetra::NOXSolver " <<
                 "Invalid value for parameter \" Jacobian Operator\"= " <<
                 jacobianSource << std::endl);

  // Create separate preconditioner if the model supports it
  /* NOTE: How do we want to decide between using an
   * available preconditioner: (1) If the model supports
   * it, then we use it, or (2) if a parameter list says
   * User_Defined ?  [Below, logic is ooption (1).]
   */
  Teuchos::RCP<EpetraExt::ModelEvaluator::Preconditioner> WPrec;
  if (outargs.supports(EpetraExt::ModelEvaluator::OUT_ARG_WPrec))
    WPrec = model->create_WPrec(); 

  // Create the linear system
  if (custom_linsys != Teuchos::null)
    linsys = custom_linsys;
  else {
    if (WPrec != Teuchos::null) {
      Teuchos::RCP<NOX::Epetra::Interface::Preconditioner> iPrec = interface;
      linsys = Teuchos::rcp(new NOX::Epetra::LinearSystemStratimikos(
			      printParams,
			      noxstratlsParams, iJac, A, iPrec, WPrec->PrecOp,
			      *currentSolution, WPrec->isAlreadyInverted));
    }
    else {
      linsys = Teuchos::rcp(new NOX::Epetra::LinearSystemStratimikos(
			      printParams,
			      noxstratlsParams, iJac, A, *currentSolution));
    }
  }

  // Build NOX group
  grp = Teuchos::rcp(new NOX::Epetra::Group(printParams, iReq,
					    *currentSolution, linsys));

  // Saves one resid calculation per solve, but not as safe
  if (leanMatrixFree) grp->disableLinearResidualComputation(true);

  // Create the Solver convergence test
  Teuchos::ParameterList& statusParams = noxParams->sublist("Status Tests");
  Teuchos::RCP<NOX::StatusTest::Generic> statusTests =
    NOX::StatusTest::buildStatusTests(statusParams, utils);

  // Create the solver
  solver = NOX::Solver::buildSolver(grp, statusTests, noxParams);

  // Create transpose linear solver
  Teuchos::RCP<LOCA::Abstract::Factory> epetraFactory =
    Teuchos::rcp(new LOCA::Epetra::Factory);
  globalData = LOCA::createGlobalData(piroParams, epetraFactory);
  LOCA::Epetra::TransposeLinearSystem::Factory tls_factory(globalData);
  tls_strategy = tls_factory.create(piroParams, linsys);
}