void 
Stokhos::MPInverseModelEvaluator::evalModel(const InArgs& inArgs,
					    const OutArgs& outArgs) const
{
  // Create underlying inargs
  InArgs me_inargs = me->createInArgs();

  // Pass parameters
  for (int i=0; i<num_p; i++)
    me_inargs.set_p(i, inArgs.get_p(i));

  // Pass MP parameters
  for (int i=0; i<num_p_mp; i++) {
    mp_const_vector_t p_mp = inArgs.get_p_mp(mp_p_index_map[i]);
    if (p_mp != Teuchos::null) {
      me_inargs.set_p(i+num_p, p_mp->getBlockVector());
    }
  }

  // Create underlying outargs
  OutArgs me_outargs = me->createOutArgs();


  // MP Responses
  for (int i=0; i<num_g_mp; i++) {
    int ii = mp_g_index_map[i];

    // g
    mp_vector_t g_mp = outArgs.get_g_mp(ii);
    if (g_mp != Teuchos::null) {
      me_outargs.set_g(i, Teuchos::rcp_dynamic_cast<Epetra_Vector>(g_mp->getBlockVector()));
    }

    // dg/dp
    for (int j=0; j<num_p; j++) {
      if (!outArgs.supports(OUT_ARG_DgDp_mp, ii, j).none()) {
	MPDerivative dgdp_mp = outArgs.get_DgDp_mp(ii,j);
	Teuchos::RCP<Stokhos::ProductEpetraMultiVector> dgdp_mp_mv = 
	  dgdp_mp.getMultiVector();
	Teuchos::RCP<Epetra_Operator> dgdp_mp_op =
	  dgdp_mp.getLinearOp();
	if (dgdp_mp_mv != Teuchos::null) {
	  me_outargs.set_DgDp(
	    i, j, Derivative(dgdp_mp_mv->getBlockMultiVector(),
			     dgdp_mp.getMultiVectorOrientation()));
	}
	else if (dgdp_mp_op != Teuchos::null) {
	  me_outargs.set_DgDp(i, j, Derivative(dgdp_mp_op));
	}
      }
    }

  }

  // Compute the functions
  me->evalModel(me_inargs, me_outargs);

}
void
twoD_diffusion_ME::
evalModel(const InArgs& inArgs, const OutArgs& outArgs) const
{

  //
  // Determinisic calculation
  //

  // Solution vector
  Teuchos::RCP<const Epetra_Vector> det_x = inArgs.get_x();

  // Parameters
  Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(0);
  if (p == Teuchos::null)
    p = p_init;

  Teuchos::RCP<Epetra_Vector> f = outArgs.get_f();
  Teuchos::RCP<Epetra_Operator> W = outArgs.get_W();
  Teuchos::RCP<Epetra_Operator> WPrec = outArgs.get_WPrec();
  if (f != Teuchos::null || W != Teuchos::null || WPrec != Teuchos::null) {
    if (basis != Teuchos::null) {
      for (int i=0; i<point.size(); i++)
        point[i] = (*p)[i];
      basis->evaluateBases(point, basis_vals);
      A->PutScalar(0.0);
      for (int k=0;k<A_k.size();k++)
        EpetraExt::MatrixMatrix::Add((*A_k[k]), false, basis_vals[k], *A, 1.0);
    }
    else {
      *A = *(A_k[0]);
      for (int k=1;k<A_k.size();k++)
        EpetraExt::MatrixMatrix::Add((*A_k[k]), false, (*p)[k-1], *A, 1.0);
    }
    A->FillComplete();
    A->OptimizeStorage();
  }

  // Residual
  if (f != Teuchos::null) {
    Teuchos::RCP<Epetra_Vector> kx = Teuchos::rcp(new Epetra_Vector(*x_map));
    A->Apply(*det_x,*kx);
    f->Update(1.0,*kx,-1.0, *b, 0.0);
  }

  // Jacobian
  if (W != Teuchos::null) {
    Teuchos::RCP<Epetra_CrsMatrix> jac =
      Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W, true);
    *jac = *A;
    jac->FillComplete();
    jac->OptimizeStorage();
  }

  // Preconditioner
  if (WPrec != Teuchos::null)
    precFactory->recompute(A, WPrec);

  // Responses (mean value)
  Teuchos::RCP<Epetra_Vector> g = outArgs.get_g(0);
  if (g != Teuchos::null) {
    (det_x->MeanValue(&(*g)[0]));
    (*g)[0] *= double(det_x->GlobalLength()) / double(mesh.size());
  }

  //
  // Stochastic Galerkin calculation
  //

  // Stochastic solution vector
  InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();

  // Stochastic parameters
  InArgs::sg_const_vector_t p_sg = inArgs.get_p_sg(0);

  // Stochastic residual
  OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
  if (f_sg != Teuchos::null) {

    // Get stochastic expansion data
    Teuchos::RCP<Stokhos::OrthogPolyExpansion<int,double> > expn =
      inArgs.get_sg_expansion();
    typedef Stokhos::Sparse3Tensor<int,double> Cijk_type;
    Teuchos::RCP<const Cijk_type> Cijk = expn->getTripleProduct();
    const Teuchos::Array<double>& norms = basis->norm_squared();

    if (sg_kx_vec_all.size() != basis->size()) {
      sg_kx_vec_all.resize(basis->size());
      for (int i=0;i<basis->size();i++) {
        sg_kx_vec_all[i] = Teuchos::rcp(new Epetra_Vector(*x_map));
      }
    }
    f_sg->init(0.0);

    Cijk_type::k_iterator k_begin = Cijk->k_begin();
    Cijk_type::k_iterator k_end = Cijk->k_end();
    for (Cijk_type::k_iterator k_it=k_begin; k_it!=k_end; ++k_it) {
      int k = Stokhos::index(k_it);
      for (Cijk_type::kj_iterator j_it = Cijk->j_begin(k_it);
           j_it != Cijk->j_end(k_it); ++j_it) {
        int j = Stokhos::index(j_it);
        A_k[k]->Apply((*x_sg)[j],*(sg_kx_vec_all[j]));
      }
      for (Cijk_type::kj_iterator j_it = Cijk->j_begin(k_it);
           j_it != Cijk->j_end(k_it); ++j_it) {
        int j = Stokhos::index(j_it);
        for (Cijk_type::kji_iterator i_it = Cijk->i_begin(j_it);
             i_it != Cijk->i_end(j_it); ++i_it) {
          int i = Stokhos::index(i_it);
          double c = Stokhos::value(i_it);  // C(i,j,k)
          (*f_sg)[i].Update(1.0*c/norms[i],*(sg_kx_vec_all[j]),1.0);
        }
      }
    } //End
    (*f_sg)[0].Update(-1.0,*b,1.0);
  }

  // Stochastic Jacobian
  OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
  if (W_sg != Teuchos::null) {
    for (int i=0; i<W_sg->size(); i++) {
      Teuchos::RCP<Epetra_CrsMatrix> jac =
        Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(i), true);
      *jac = *A_k[i];
      jac->FillComplete();
      jac->OptimizeStorage();
    }
  }

  // Stochastic responses
  Teuchos::RCP< Stokhos::EpetraVectorOrthogPoly > g_sg =
    outArgs.get_g_sg(0);
  if (g_sg != Teuchos::null) {
    int sz = x_sg->size();
    for (int i=0; i<sz; i++) {
      (*x_sg)[i].MeanValue(&(*g_sg)[i][0]);
      (*g_sg)[i][0] *= double((*x_sg)[i].GlobalLength()) / double(mesh.size());
    }
  }

  //
  // Multi-point calculation
  //

  // Stochastic solution vector
  mp_const_vector_t x_mp = inArgs.get_x_mp();

  // Stochastic parameters
  mp_const_vector_t p_mp = inArgs.get_p_mp(0);

  // Stochastic residual
  mp_vector_t f_mp = outArgs.get_f_mp();
  mp_operator_t W_mp = outArgs.get_W_mp();
  if (f_mp != Teuchos::null || W_mp != Teuchos::null) {
    int num_mp = x_mp->size();
    for (int i=0; i<num_mp; i++) {
      // Compute operator
      if (basis != Teuchos::null) {
        for (int k=0; k<point.size(); k++)
          point[k] = (*p_mp)[i][k];
        basis->evaluateBases(point, basis_vals);
        A->PutScalar(0.0);
        for (int k=0;k<A_k.size();k++)
          EpetraExt::MatrixMatrix::Add((*A_k[k]), false, basis_vals[k], *A,
                                       1.0);
      }
      else {
        *A = *(A_k[0]);
        for (int k=1;k<A_k.size();k++)
          EpetraExt::MatrixMatrix::Add((*A_k[k]), false, (*p_mp)[i][k-1], *A,
                                       1.0);
      }

      A->FillComplete();
      A->OptimizeStorage();

      // Compute residual
      if (f_mp != Teuchos::null) {
        A->Apply((*x_mp)[i], (*f_mp)[i]);
        (*f_mp)[i].Update(-1.0, *b, 1.0);
      }

      // Copy operator
      if (W_mp != Teuchos::null) {
        Teuchos::RCP<Epetra_CrsMatrix> jac =
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_mp->getCoeffPtr(i),
                                                      true);
        *jac = *A;
        jac->FillComplete();
        jac->OptimizeStorage();
      }
    }
  }

  // Multipoint responses
  mp_vector_t g_mp = outArgs.get_g_mp(0);
  if (g_mp != Teuchos::null) {
    int sz = x_mp->size();
    for (int i=0; i<sz; i++) {
      (*x_mp)[i].MeanValue(&(*g_mp)[i][0]);
      (*g_mp)[i][0] *= double((*x_mp)[i].GlobalLength()) / double(mesh.size());
    }
  }

}
Example #3
0
void
Albany::ModelEvaluator::evalModel(const InArgs& inArgs,
                                 const OutArgs& outArgs) const
{
  Teuchos::TimeMonitor Timer(*timer); //start timer
  //
  // Get the input arguments
  //
  Teuchos::RCP<const Epetra_Vector> x = inArgs.get_x();
  Teuchos::RCP<const Epetra_Vector> x_dot;
  Teuchos::RCP<const Epetra_Vector> x_dotdot;
  double alpha     = 0.0;
  double omega     = 0.0;
  double beta      = 1.0;
  double curr_time = 0.0;
  x_dot = inArgs.get_x_dot();
  x_dotdot = inArgs.get_x_dotdot();
  if (x_dot != Teuchos::null || x_dotdot != Teuchos::null) {
    alpha = inArgs.get_alpha();
    omega = inArgs.get_omega();
    beta = inArgs.get_beta();
    curr_time  = inArgs.get_t();
  }
  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i);
    if (p != Teuchos::null) {
      for (unsigned int j=0; j<sacado_param_vec[i].size(); j++)
        sacado_param_vec[i][j].baseValue = (*p)[j];
    }
  }
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i+num_param_vecs);
    if (p != Teuchos::null) {
      *(distParamLib->get(dist_param_names[i])->vector()) = *p;
    }
  }

  //
  // Get the output arguments
  //
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> f_out = outArgs.get_f();
  Teuchos::RCP<Epetra_Operator> W_out = outArgs.get_W();

  // Cast W to a CrsMatrix, throw an exception if this fails
  Teuchos::RCP<Epetra_CrsMatrix> W_out_crs;

  if (W_out != Teuchos::null)
    W_out_crs = Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);

int test_var = 0;
if(test_var != 0){
std::cout << "The current solution length is: " << x->MyLength() << std::endl;
x->Print(std::cout);

}

  // Get preconditioner operator, if requested
  Teuchos::RCP<Epetra_Operator> WPrec_out;
  if (outArgs.supports(OUT_ARG_WPrec)) WPrec_out = outArgs.get_WPrec();

  //
  // Compute the functions
  //
  bool f_already_computed = false;

  // W matrix
  if (W_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(),*x,
                               sacado_param_vec, f_out.get(), *W_out_crs);
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current Jacobian length is: " << W_out_crs->NumGlobalRows() << std::endl;
W_out_crs->Print(std::cout);
}
  }

  if (WPrec_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(), *x,
                               sacado_param_vec, f_out.get(), *Extra_W_crs);
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current preconditioner length is: " << Extra_W_crs->NumGlobalRows() << std::endl;
Extra_W_crs->Print(std::cout);
}

    app->computeGlobalPreconditioner(Extra_W_crs, WPrec_out);
  }

  // scalar df/dp
  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<Epetra_MultiVector> dfdp_out =
      outArgs.get_DfDp(i).getMultiVector();
    if (dfdp_out != Teuchos::null) {
      Teuchos::Array<int> p_indexes =
        outArgs.get_DfDp(i).getDerivativeMultiVector().getParamIndexes();
      Teuchos::RCP<ParamVec> p_vec;
      if (p_indexes.size() == 0)
        p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
      else {
        p_vec = Teuchos::rcp(new ParamVec);
        for (int j=0; j<p_indexes.size(); j++)
          p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                          sacado_param_vec[i][p_indexes[j]].baseValue);
      }

      app->computeGlobalTangent(0.0, 0.0, 0.0, curr_time, false, x_dot.get(), x_dotdot.get(), *x,
                                sacado_param_vec, p_vec.get(),
                                NULL, NULL, NULL, NULL, f_out.get(), NULL,
                                dfdp_out.get());

      f_already_computed=true;
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }

  // distributed df/dp
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<Epetra_Operator> dfdp_out =
      outArgs.get_DfDp(i+num_param_vecs).getLinearOp();
    if (dfdp_out != Teuchos::null) {
      Teuchos::RCP<DistributedParameterDerivativeOp> dfdp_op =
        Teuchos::rcp_dynamic_cast<DistributedParameterDerivativeOp>(dfdp_out);
      dfdp_op->set(curr_time, x_dot, x_dotdot, x,
                   Teuchos::rcp(&sacado_param_vec,false));
    }
  }

  // f
  if (app->is_adjoint) {
    Derivative f_deriv(f_out, DERIV_TRANS_MV_BY_ROW);
    int response_index = 0; // need to add capability for sending this in
    app->evaluateResponseDerivative(response_index, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                    sacado_param_vec, NULL,
                                    NULL, f_deriv, Derivative(), Derivative(), Derivative());
  }
  else {
    if (f_out != Teuchos::null && !f_already_computed) {
      app->computeGlobalResidual(curr_time, x_dot.get(), x_dotdot.get(), *x,
                                 sacado_param_vec, *f_out);
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }

  // Response functions
  for (int i=0; i<outArgs.Ng(); i++) {
    Teuchos::RCP<Epetra_Vector> g_out = outArgs.get_g(i);
    bool g_computed = false;

    Derivative dgdx_out = outArgs.get_DgDx(i);
    Derivative dgdxdot_out = outArgs.get_DgDx_dot(i);
    Derivative dgdxdotdot_out = outArgs.get_DgDx_dotdot(i);

    // dg/dx, dg/dxdot
    if (!dgdx_out.isEmpty() || !dgdxdot_out.isEmpty() || !dgdxdotdot_out.isEmpty() ) {
      app->evaluateResponseDerivative(i, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                      sacado_param_vec, NULL,
                                      g_out.get(), dgdx_out,
                                      dgdxdot_out, dgdxdotdot_out, Derivative());
      g_computed = true;
    }

    // dg/dp
    for (int j=0; j<num_param_vecs; j++) {
      Teuchos::RCP<Epetra_MultiVector> dgdp_out =
        outArgs.get_DgDp(i,j).getMultiVector();
      if (dgdp_out != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DgDp(i,j).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int k=0; k<p_indexes.size(); k++)
            p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                            sacado_param_vec[j][p_indexes[k]].baseValue);
        }
        app->evaluateResponseTangent(i, alpha, beta, omega, curr_time, false,
                                     x_dot.get(), x_dotdot.get(), *x,
                                     sacado_param_vec, p_vec.get(),
                                     NULL, NULL, NULL, NULL, g_out.get(), NULL,
                                     dgdp_out.get());
        g_computed = true;
      }
    }

    // Need to handle dg/dp for distributed p

    if (g_out != Teuchos::null && !g_computed)
      app->evaluateResponse(i, curr_time, x_dot.get(), x_dotdot.get(), *x, sacado_param_vec,
                            *g_out);
  }

  //
  // Stochastic Galerkin
  //
#ifdef ALBANY_SG_MP
  InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();
  if (x_sg != Teuchos::null) {
    app->init_sg(inArgs.get_sg_basis(),
                 inArgs.get_sg_quadrature(),
                 inArgs.get_sg_expansion(),
                 x_sg->productComm());
    InArgs::sg_const_vector_t x_dot_sg  = inArgs.get_x_dot_sg();
    InArgs::sg_const_vector_t x_dotdot_sg  = inArgs.get_x_dotdot_sg();
    if (x_dot_sg != Teuchos::null || x_dotdot_sg != Teuchos::null) {
      alpha = inArgs.get_alpha();
      omega = inArgs.get_omega();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }

    InArgs::sg_const_vector_t epetra_p_sg = inArgs.get_p_sg(0);
    Teuchos::Array<int> p_sg_index;
    for (int i=0; i<num_param_vecs; i++) {
      InArgs::sg_const_vector_t p_sg = inArgs.get_p_sg(i);
      if (p_sg != Teuchos::null) {
        p_sg_index.push_back(i);
        for (int j=0; j<p_sg_vals[i].size(); j++) {
          int num_sg_blocks = p_sg->size();
          p_sg_vals[i][j].reset(app->getStochasticExpansion(), num_sg_blocks);
          p_sg_vals[i][j].copyForWrite();
          for (int l=0; l<num_sg_blocks; l++) {
            p_sg_vals[i][j].fastAccessCoeff(l) = (*p_sg)[l][j];
          }
        }
      }
    }

    OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
    OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
    bool f_sg_computed = false;

    // W_sg
    if (W_sg != Teuchos::null) {
      Stokhos::VectorOrthogPoly<Epetra_CrsMatrix> W_sg_crs(W_sg->basis(),
                                                           W_sg->map());
      for (int i=0; i<W_sg->size(); i++)
        W_sg_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(i)));
      app->computeGlobalSGJacobian(alpha, beta, omega, curr_time,
                                   x_dot_sg.get(),  x_dotdot_sg.get(), *x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   f_sg.get(), W_sg_crs);
      f_sg_computed = true;
    }

    // df/dp_sg
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dfdp_sg
        = outArgs.get_DfDp_sg(i).getMultiVector();
      if (dfdp_sg != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_sg(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalSGTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                    sacado_param_vec, p_sg_index, p_sg_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_sg.get(), NULL, dfdp_sg.get());

        f_sg_computed = true;
      }
    }

    if (f_sg != Teuchos::null && !f_sg_computed)
      app->computeGlobalSGResidual(curr_time, x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   *f_sg);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      OutArgs::sg_vector_t g_sg = outArgs.get_g_sg(i);
      bool g_sg_computed = false;

      SGDerivative dgdx_sg = outArgs.get_DgDx_sg(i);
      SGDerivative dgdxdot_sg = outArgs.get_DgDx_dot_sg(i);
      SGDerivative dgdxdotdot_sg = outArgs.get_DgDx_dotdot_sg(i);

      // dg/dx, dg/dxdot
      if (!dgdx_sg.isEmpty() || !dgdxdot_sg.isEmpty() || !dgdxdotdot_sg.isEmpty()) {
        app->evaluateSGResponseDerivative(
            i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
            sacado_param_vec, p_sg_index, p_sg_vals,
            NULL, g_sg.get(), dgdx_sg,
            dgdxdot_sg, dgdxdotdot_sg, SGDerivative());
        g_sg_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dgdp_sg =
          outArgs.get_DgDp_sg(i,j).getMultiVector();
        if (dgdp_sg != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_sg(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateSGResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                         sacado_param_vec, p_sg_index,
                                         p_sg_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_sg.get(),
                                         NULL, dgdp_sg.get());
          g_sg_computed = true;

        }
      }

      if (g_sg != Teuchos::null && !g_sg_computed)
        app->evaluateSGResponse(i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                sacado_param_vec, p_sg_index, p_sg_vals,
                                *g_sg);
    }
  }

  //
  // Multi-point evaluation
  //
  mp_const_vector_t x_mp = inArgs.get_x_mp();
  if (x_mp != Teuchos::null) {
    mp_const_vector_t x_dot_mp  = inArgs.get_x_dot_mp();
    mp_const_vector_t x_dotdot_mp  = inArgs.get_x_dotdot_mp();
    if (x_dot_mp != Teuchos::null || x_dotdot_mp != Teuchos::null) {
      alpha = inArgs.get_alpha();
      omega = inArgs.get_omega();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }

    Teuchos::Array<int> p_mp_index;
    for (int i=0; i<num_param_vecs; i++) {
      mp_const_vector_t p_mp = inArgs.get_p_mp(i);
      if (p_mp != Teuchos::null) {
        p_mp_index.push_back(i);
        for (int j=0; j<p_mp_vals[i].size(); j++) {
          int num_mp_blocks = p_mp->size();
          p_mp_vals[i][j].reset(num_mp_blocks);
          p_mp_vals[i][j].copyForWrite();
          for (int l=0; l<num_mp_blocks; l++) {
            p_mp_vals[i][j].fastAccessCoeff(l) = (*p_mp)[l][j];
          }
        }
      }
    }

    mp_vector_t f_mp = outArgs.get_f_mp();
    mp_operator_t W_mp = outArgs.get_W_mp();
    bool f_mp_computed = false;

    // W_mp
    if (W_mp != Teuchos::null) {
      Stokhos::ProductContainer<Epetra_CrsMatrix> W_mp_crs(W_mp->map());
      for (int i=0; i<W_mp->size(); i++)
        W_mp_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_mp->getCoeffPtr(i)));
      app->computeGlobalMPJacobian(alpha, beta, omega, curr_time,
                                   x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   f_mp.get(), W_mp_crs);
      f_mp_computed = true;
    }

    // df/dp_mp
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dfdp_mp
        = outArgs.get_DfDp_mp(i).getMultiVector();
      if (dfdp_mp != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_mp(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalMPTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                    sacado_param_vec, p_mp_index, p_mp_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_mp.get(), NULL, dfdp_mp.get());

        f_mp_computed = true;
      }
    }

    if (f_mp != Teuchos::null && !f_mp_computed)
      app->computeGlobalMPResidual(curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   *f_mp);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      mp_vector_t g_mp = outArgs.get_g_mp(i);
      bool g_mp_computed = false;

      MPDerivative dgdx_mp = outArgs.get_DgDx_mp(i);
      MPDerivative dgdxdot_mp = outArgs.get_DgDx_dot_mp(i);
      MPDerivative dgdxdotdot_mp = outArgs.get_DgDx_dotdot_mp(i);

      // dg/dx, dg/dxdot
      if (!dgdx_mp.isEmpty() || !dgdxdot_mp.isEmpty() || !dgdxdotdot_mp.isEmpty() ) {
        app->evaluateMPResponseDerivative(
            i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
            sacado_param_vec, p_mp_index, p_mp_vals,
            NULL, g_mp.get(), dgdx_mp,
            dgdxdot_mp, dgdxdotdot_mp, MPDerivative());
        g_mp_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dgdp_mp =
          outArgs.get_DgDp_mp(i,j).getMultiVector();
        if (dgdp_mp != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_mp(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateMPResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                         sacado_param_vec, p_mp_index,
                                         p_mp_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_mp.get(),
                                         NULL, dgdp_mp.get());
          g_mp_computed = true;
        }
      }

      if (g_mp != Teuchos::null && !g_mp_computed)
        app->evaluateMPResponse(i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                sacado_param_vec, p_mp_index, p_mp_vals,
                                *g_mp);
    }
  }
#endif //ALBANY_SG_MP
}
void
Albany::ModelEvaluator::evalModel(const InArgs& inArgs,
                                 const OutArgs& outArgs) const
{
  Teuchos::TimeMonitor Timer(*timer); //start timer
  //
  // Get the input arguments
  //
  Teuchos::RCP<const Epetra_Vector> x = inArgs.get_x();
  Teuchos::RCP<const Epetra_Vector> x_dot;
  Teuchos::RCP<const Epetra_Vector> x_dotdot;

  //create comm and node objects for Epetra -> Tpetra conversions
  Teuchos::RCP<const Teuchos::Comm<int> > commT = app->getComm();
  Teuchos::RCP<Epetra_Comm> comm = Albany::createEpetraCommFromTeuchosComm(commT);
  //Create Tpetra copy of x, call it xT
  Teuchos::RCP<const Tpetra_Vector> xT;
  if (x != Teuchos::null)
    xT  = Petra::EpetraVector_To_TpetraVectorConst(*x, commT);

  double alpha     = 0.0;
  double omega     = 0.0;
  double beta      = 1.0;
  double curr_time = 0.0;

  if(num_time_deriv > 0)
    x_dot = inArgs.get_x_dot();
  if(num_time_deriv > 1)
    x_dotdot = inArgs.get_x_dotdot();

  //Declare and create Tpetra copy of x_dot, call it x_dotT
  Teuchos::RCP<const Tpetra_Vector> x_dotT;
  if (Teuchos::nonnull(x_dot))
    x_dotT = Petra::EpetraVector_To_TpetraVectorConst(*x_dot, commT);

  //Declare and create Tpetra copy of x_dotdot, call it x_dotdotT
  Teuchos::RCP<const Tpetra_Vector> x_dotdotT;
  if (Teuchos::nonnull(x_dotdot))
    x_dotdotT = Petra::EpetraVector_To_TpetraVectorConst(*x_dotdot, commT);

  if (Teuchos::nonnull(x_dot)){
    alpha = inArgs.get_alpha();
    beta = inArgs.get_beta();
    curr_time  = inArgs.get_t();
  }
  if (Teuchos::nonnull(x_dotdot)) {
    omega = inArgs.get_omega();
  }

  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i);
    if (p != Teuchos::null) {
      for (unsigned int j=0; j<sacado_param_vec[i].size(); j++) {
        sacado_param_vec[i][j].baseValue = (*p)[j];
      }
    }
  }
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i+num_param_vecs);
    //create Tpetra copy of p
    Teuchos::RCP<const Tpetra_Vector> pT;
    if (p != Teuchos::null) {
      pT = Petra::EpetraVector_To_TpetraVectorConst(*p, commT);
      //*(distParamLib->get(dist_param_names[i])->vector()) = *p;
      *(distParamLib->get(dist_param_names[i])->vector()) = *pT;
    }
  }

  //
  // Get the output arguments
  //
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> f_out = outArgs.get_f();
  Teuchos::RCP<Epetra_Operator> W_out = outArgs.get_W();

  // Cast W to a CrsMatrix, throw an exception if this fails
  Teuchos::RCP<Epetra_CrsMatrix> W_out_crs;
#ifdef WRITE_MASS_MATRIX_TO_MM_FILE
  //IK, 7/15/14: adding object to hold mass matrix to be written to matrix market file
  Teuchos::RCP<Epetra_CrsMatrix> Mass;
  //IK, 7/15/14: needed for writing mass matrix out to matrix market file
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> ftmp = outArgs.get_f();
#endif

  if (W_out != Teuchos::null) {
    W_out_crs = Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);
#ifdef WRITE_MASS_MATRIX_TO_MM_FILE
    //IK, 7/15/14: adding object to hold mass matrix to be written to matrix market file
    Mass = Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);
#endif
  }


int test_var = 0;
if(test_var != 0){
std::cout << "The current solution length is: " << x->MyLength() << std::endl;
x->Print(std::cout);

}

  // Get preconditioner operator, if requested
  Teuchos::RCP<Epetra_Operator> WPrec_out;
  if (outArgs.supports(OUT_ARG_WPrec)) WPrec_out = outArgs.get_WPrec();

  //
  // Compute the functions
  //
  bool f_already_computed = false;

  // W matrix
  if (W_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(),*x,
                               sacado_param_vec, f_out.get(), *W_out_crs);
#ifdef WRITE_MASS_MATRIX_TO_MM_FILE
    //IK, 7/15/14: write mass matrix to matrix market file
    //Warning: to read this in to MATLAB correctly, code must be run in serial.
    //Otherwise Mass will have a distributed Map which would also need to be read in to MATLAB for proper
    //reading in of Mass.
    app->computeGlobalJacobian(1.0, 0.0, 0.0, curr_time, x_dot.get(), x_dotdot.get(), *x,
                               sacado_param_vec, ftmp.get(), *Mass);
    EpetraExt::RowMatrixToMatrixMarketFile("mass.mm", *Mass);
    EpetraExt::BlockMapToMatrixMarketFile("rowmap.mm", Mass->RowMap());
    EpetraExt::BlockMapToMatrixMarketFile("colmap.mm", Mass->ColMap());
    Teuchos::RCP<Teuchos::FancyOStream> out = Teuchos::VerboseObjectBase::getDefaultOStream();
#endif
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current Jacobian length is: " << W_out_crs->NumGlobalRows() << std::endl;
W_out_crs->Print(std::cout);
}
  }

  if (WPrec_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(), *x,
                               sacado_param_vec, f_out.get(), *Extra_W_crs);
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current preconditioner length is: " << Extra_W_crs->NumGlobalRows() << std::endl;
Extra_W_crs->Print(std::cout);
}

    app->computeGlobalPreconditioner(Extra_W_crs, WPrec_out);
  }

  // scalar df/dp
  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<Epetra_MultiVector> dfdp_out =
      outArgs.get_DfDp(i).getMultiVector();
    if (dfdp_out != Teuchos::null) {
      Teuchos::Array<int> p_indexes =
        outArgs.get_DfDp(i).getDerivativeMultiVector().getParamIndexes();
      Teuchos::RCP<ParamVec> p_vec;
      if (p_indexes.size() == 0)
        p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
      else {
        p_vec = Teuchos::rcp(new ParamVec);
        for (int j=0; j<p_indexes.size(); j++)
          p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                          sacado_param_vec[i][p_indexes[j]].baseValue);
      }

      app->computeGlobalTangent(0.0, 0.0, 0.0, curr_time, false, x_dot.get(), x_dotdot.get(), *x,
                                sacado_param_vec, p_vec.get(),
                                NULL, NULL, NULL, NULL, f_out.get(), NULL,
                                dfdp_out.get());

      f_already_computed=true;
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }

  // distributed df/dp
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<Epetra_Operator> dfdp_out =
      outArgs.get_DfDp(i+num_param_vecs).getLinearOp();
    if (dfdp_out != Teuchos::null) {
      Teuchos::RCP<DistributedParameterDerivativeOp> dfdp_op =
        Teuchos::rcp_dynamic_cast<DistributedParameterDerivativeOp>(dfdp_out);
      dfdp_op->set(curr_time, x_dotT, x_dotdotT, xT,
                   Teuchos::rcp(&sacado_param_vec,false));
    }
  }

  // f
  if (app->is_adjoint) {
    Derivative f_deriv(f_out, DERIV_TRANS_MV_BY_ROW);
    int response_index = 0; // need to add capability for sending this in
    app->evaluateResponseDerivative(response_index, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                    sacado_param_vec, NULL,
                                    NULL, f_deriv, Derivative(), Derivative(), Derivative());
  }
  else {
    if (f_out != Teuchos::null && !f_already_computed) {
      app->computeGlobalResidual(curr_time, x_dot.get(), x_dotdot.get(), *x,
                                  sacado_param_vec, *f_out);
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }


  // Response functions
  for (int i=0; i<outArgs.Ng(); i++) {
    //Set curr_time to final time at which response occurs.
    if(num_time_deriv > 0)
      curr_time  = inArgs.get_t();
    Teuchos::RCP<Epetra_Vector> g_out = outArgs.get_g(i);
    //Declare Tpetra_Vector copy of g_out
    Teuchos::RCP<Tpetra_Vector> g_outT;
    bool g_computed = false;

    Derivative dgdx_out = outArgs.get_DgDx(i);
    Derivative dgdxdot_out = outArgs.get_DgDx_dot(i);
    Derivative dgdxdotdot_out = outArgs.get_DgDx_dotdot(i);

    // dg/dx, dg/dxdot
    if (!dgdx_out.isEmpty() || !dgdxdot_out.isEmpty() || !dgdxdotdot_out.isEmpty() ) {
      app->evaluateResponseDerivative(i, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                      sacado_param_vec, NULL,
                                      g_out.get(), dgdx_out,
                                      dgdxdot_out, dgdxdotdot_out, Derivative());
      g_computed = true;
    }

    // dg/dp
    for (int j=0; j<num_param_vecs; j++) {
      Teuchos::RCP<Epetra_MultiVector> dgdp_out =
        outArgs.get_DgDp(i,j).getMultiVector();
      //Declare Tpetra copy of dgdp_out
      Teuchos::RCP<Tpetra_MultiVector> dgdp_outT;
      if (dgdp_out != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DgDp(i,j).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int k=0; k<p_indexes.size(); k++)
            p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                            sacado_param_vec[j][p_indexes[k]].baseValue);
        }
        //create Tpetra copy of g_out, call it g_outT
        if (g_out != Teuchos::null)
           g_outT = Petra::EpetraVector_To_TpetraVectorNonConst(*g_out, commT);
        //create Tpetra copy of dgdp_out, call it dgdp_outT
        if (dgdp_out != Teuchos::null)
           dgdp_outT = Petra::EpetraMultiVector_To_TpetraMultiVector(*dgdp_out, commT);
	app->evaluateResponseTangentT(i, alpha, beta, omega, curr_time, false,
				     x_dotT.get(), x_dotdotT.get(), *xT,
				     sacado_param_vec, p_vec.get(),
				     NULL, NULL, NULL, NULL, g_outT.get(), NULL,
				     dgdp_outT.get());
        //convert g_outT to Epetra_Vector g_out
        if (g_out != Teuchos::null)
          Petra::TpetraVector_To_EpetraVector(g_outT, *g_out, comm);
        //convert dgdp_outT to Epetra_MultiVector dgdp_out
        if (dgdp_out != Teuchos::null)
          Petra::TpetraMultiVector_To_EpetraMultiVector(dgdp_outT, *dgdp_out, comm);
        g_computed = true;
      }
    }

    // Need to handle dg/dp for distributed p
    for(int j=0; j<num_dist_param_vecs; j++) {
      Derivative dgdp_out = outArgs.get_DgDp(i,j+num_param_vecs);
      if (!dgdp_out.isEmpty()) {
        dgdp_out.getMultiVector()->PutScalar(0.);
        app->evaluateResponseDistParamDeriv(i, curr_time, x_dot.get(), x_dotdot.get(), *x, sacado_param_vec, dist_param_names[j], dgdp_out.getMultiVector().get());
      }
    }

    if (g_out != Teuchos::null && !g_computed) {
      //create Tpetra copy of g_out, call it g_outT
      g_outT = Petra::EpetraVector_To_TpetraVectorNonConst(*g_out, commT);
      app->evaluateResponseT(i, curr_time, x_dotT.get(), x_dotdotT.get(), *xT, sacado_param_vec,
			    *g_outT);
      //convert g_outT to Epetra_Vector g_out
      Petra::TpetraVector_To_EpetraVector(g_outT, *g_out, comm);
    }
  }

  //
  // Stochastic Galerkin
  //
#ifdef ALBANY_SG
  InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();
  if (x_sg != Teuchos::null) {
    app->init_sg(inArgs.get_sg_basis(),
                 inArgs.get_sg_quadrature(),
                 inArgs.get_sg_expansion(),
                 x_sg->productComm());
    InArgs::sg_const_vector_t x_dot_sg  = Teuchos::null;
    InArgs::sg_const_vector_t x_dot_sg  = Teuchos::null;
    if(num_time_deriv > 0)
      x_dotdot_sg  = inArgs.get_x_dotdot_sg();
    if(num_time_deriv > 1)
      x_dotdot_sg  = inArgs.get_x_dotdot_sg();
    if (x_dot_sg != Teuchos::null || x_dotdot_sg != Teuchos::null) {
      alpha = inArgs.get_alpha();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }
    if (x_dotdot_sg != Teuchos::null) {
      omega = inArgs.get_omega();
    }

    InArgs::sg_const_vector_t epetra_p_sg = inArgs.get_p_sg(0);
    Teuchos::Array<int> p_sg_index;
    for (int i=0; i<num_param_vecs; i++) {
      InArgs::sg_const_vector_t p_sg = inArgs.get_p_sg(i);
      if (p_sg != Teuchos::null) {
        p_sg_index.push_back(i);
        for (int j=0; j<p_sg_vals[i].size(); j++) {
          int num_sg_blocks = p_sg->size();
          p_sg_vals[i][j].reset(app->getStochasticExpansion(), num_sg_blocks);
          p_sg_vals[i][j].copyForWrite();
          for (int l=0; l<num_sg_blocks; l++) {
            p_sg_vals[i][j].fastAccessCoeff(l) = (*p_sg)[l][j];
          }
        }
      }
    }

    OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
    OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
    bool f_sg_computed = false;

    // W_sg
    if (W_sg != Teuchos::null) {
      Stokhos::VectorOrthogPoly<Epetra_CrsMatrix> W_sg_crs(W_sg->basis(),
                                                           W_sg->map());
      for (int i=0; i<W_sg->size(); i++)
        W_sg_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(i)));
      app->computeGlobalSGJacobian(alpha, beta, omega, curr_time,
                                   x_dot_sg.get(),  x_dotdot_sg.get(), *x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   f_sg.get(), W_sg_crs);
      f_sg_computed = true;
    }

    // df/dp_sg
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dfdp_sg
        = outArgs.get_DfDp_sg(i).getMultiVector();
      if (dfdp_sg != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_sg(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalSGTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                    sacado_param_vec, p_sg_index, p_sg_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_sg.get(), NULL, dfdp_sg.get());

        f_sg_computed = true;
      }
    }

    if (f_sg != Teuchos::null && !f_sg_computed)
      app->computeGlobalSGResidual(curr_time, x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   *f_sg);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      OutArgs::sg_vector_t g_sg = outArgs.get_g_sg(i);
      bool g_sg_computed = false;

      SGDerivative dgdx_sg = outArgs.get_DgDx_sg(i);
      SGDerivative dgdxdot_sg = outArgs.get_DgDx_dot_sg(i);
      SGDerivative dgdxdotdot_sg = outArgs.get_DgDx_dotdot_sg(i);

      // dg/dx, dg/dxdot
      if (!dgdx_sg.isEmpty() || !dgdxdot_sg.isEmpty() || !dgdxdotdot_sg.isEmpty()) {
        app->evaluateSGResponseDerivative(
            i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
            sacado_param_vec, p_sg_index, p_sg_vals,
            NULL, g_sg.get(), dgdx_sg,
            dgdxdot_sg, dgdxdotdot_sg, SGDerivative());
        g_sg_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dgdp_sg =
          outArgs.get_DgDp_sg(i,j).getMultiVector();
        if (dgdp_sg != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_sg(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateSGResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                         sacado_param_vec, p_sg_index,
                                         p_sg_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_sg.get(),
                                         NULL, dgdp_sg.get());
          g_sg_computed = true;

        }
      }

      if (g_sg != Teuchos::null && !g_sg_computed)
        app->evaluateSGResponse(i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                sacado_param_vec, p_sg_index, p_sg_vals,
                                *g_sg);
    }
  }
#endif
#ifdef ALBANY_ENSEMBLE

  //
  // Multi-point evaluation
  //
  mp_const_vector_t x_mp = inArgs.get_x_mp();
  if (x_mp != Teuchos::null) {
    mp_const_vector_t x_dot_mp  = Teuchos::null;
    mp_const_vector_t x_dotdot_mp  = Teuchos::null;
    if(num_time_deriv > 0)
      x_dot_mp  = inArgs.get_x_dot_mp();
    if(num_time_deriv > 1)
      x_dotdot_mp  = inArgs.get_x_dotdot_mp();
    if (x_dot_mp != Teuchos::null || x_dotdot_mp != Teuchos::null) {
      alpha = inArgs.get_alpha();
      //omega = inArgs.get_omega();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }
    if (x_dotdot_mp != Teuchos::null) {
      omega = inArgs.get_omega();
    }

    Teuchos::Array<int> p_mp_index;
    for (int i=0; i<num_param_vecs; i++) {
      mp_const_vector_t p_mp = inArgs.get_p_mp(i);
      if (p_mp != Teuchos::null) {
        p_mp_index.push_back(i);
        for (int j=0; j<p_mp_vals[i].size(); j++) {
          int num_mp_blocks = p_mp->size();
          p_mp_vals[i][j].reset(num_mp_blocks);
          p_mp_vals[i][j].copyForWrite();
          for (int l=0; l<num_mp_blocks; l++) {
            p_mp_vals[i][j].fastAccessCoeff(l) = (*p_mp)[l][j];
          }
        }
      }
    }

    mp_vector_t f_mp = outArgs.get_f_mp();
    mp_operator_t W_mp = outArgs.get_W_mp();
    bool f_mp_computed = false;

    // W_mp
    if (W_mp != Teuchos::null) {
      Stokhos::ProductContainer<Epetra_CrsMatrix> W_mp_crs(W_mp->map());
      for (int i=0; i<W_mp->size(); i++)
        W_mp_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_mp->getCoeffPtr(i)));
      app->computeGlobalMPJacobian(alpha, beta, omega, curr_time,
                                   x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   f_mp.get(), W_mp_crs);
      f_mp_computed = true;
    }

    // df/dp_mp
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dfdp_mp
        = outArgs.get_DfDp_mp(i).getMultiVector();
      if (dfdp_mp != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_mp(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalMPTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                    sacado_param_vec, p_mp_index, p_mp_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_mp.get(), NULL, dfdp_mp.get());

        f_mp_computed = true;
      }
    }

    if (f_mp != Teuchos::null && !f_mp_computed)
      app->computeGlobalMPResidual(curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   *f_mp);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      mp_vector_t g_mp = outArgs.get_g_mp(i);
      bool g_mp_computed = false;

      MPDerivative dgdx_mp = outArgs.get_DgDx_mp(i);
      MPDerivative dgdxdot_mp = outArgs.get_DgDx_dot_mp(i);
      MPDerivative dgdxdotdot_mp = outArgs.get_DgDx_dotdot_mp(i);

      // dg/dx, dg/dxdot
      if (!dgdx_mp.isEmpty() || !dgdxdot_mp.isEmpty() || !dgdxdotdot_mp.isEmpty() ) {
        app->evaluateMPResponseDerivative(
            i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
            sacado_param_vec, p_mp_index, p_mp_vals,
            NULL, g_mp.get(), dgdx_mp,
            dgdxdot_mp, dgdxdotdot_mp, MPDerivative());
        g_mp_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dgdp_mp =
          outArgs.get_DgDp_mp(i,j).getMultiVector();
        if (dgdp_mp != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_mp(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateMPResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                         sacado_param_vec, p_mp_index,
                                         p_mp_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_mp.get(),
                                         NULL, dgdp_mp.get());
          g_mp_computed = true;
        }
      }

      if (g_mp != Teuchos::null && !g_mp_computed)
        app->evaluateMPResponse(i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                sacado_param_vec, p_mp_index, p_mp_vals,
                                *g_mp);
    }
  }
#endif
}