void Simple_ModelEval::evalModel( const InArgs& inArgs,
                              const OutArgs& outArgs ) const
{

  // Parse InArgs
  Teuchos::RCP<const Epetra_Vector> p_in = inArgs.get_p(0);
  if (!p_in.get()) cout << "ERROR: Simple_ModelEval requires p as inargs" << endl;
  int numParameters = p_in->GlobalLength();

  // Parse OutArgs

  Teuchos::RCP<Epetra_Vector> g_out = outArgs.get_g(0); 

  // Parse out-args for sensitivity calculation
  Teuchos::RCP<Epetra_MultiVector> dgdp_out;

  dgdp_out = outArgs.get_DgDp(0,0).getMultiVector();

  if (!is_null(g_out)) {
    (*g_out)[0] = 1.0 - (*p_in)[0];
    (*g_out)[1] = 1.2 - (*p_in)[1];
    (*g_out)[2] = 4.0 - (*p_in)[2] - 0.5* (1.0 - (*p_in)[0]);
  }

  if (dgdp_out != Teuchos::null) {
     // Must initialize since Thyra will init with NaN
     dgdp_out->PutScalar(0.0);
     // Set gradient of above g equations (derived by hand)
     for (int i=0; i<numParameters; i++) {
       (*dgdp_out)[i][i] = -1.0;
     }
     (*dgdp_out)[0][2] = 0.5; //DERIV_BY_COL: [p][g]
   }
} 
void 
Stokhos::MPInverseModelEvaluator::evalModel(const InArgs& inArgs,
					    const OutArgs& outArgs) const
{
  // Create underlying inargs
  InArgs me_inargs = me->createInArgs();

  // Pass parameters
  for (int i=0; i<num_p; i++)
    me_inargs.set_p(i, inArgs.get_p(i));

  // Pass MP parameters
  for (int i=0; i<num_p_mp; i++) {
    mp_const_vector_t p_mp = inArgs.get_p_mp(mp_p_index_map[i]);
    if (p_mp != Teuchos::null) {
      me_inargs.set_p(i+num_p, p_mp->getBlockVector());
    }
  }

  // Create underlying outargs
  OutArgs me_outargs = me->createOutArgs();


  // MP Responses
  for (int i=0; i<num_g_mp; i++) {
    int ii = mp_g_index_map[i];

    // g
    mp_vector_t g_mp = outArgs.get_g_mp(ii);
    if (g_mp != Teuchos::null) {
      me_outargs.set_g(i, Teuchos::rcp_dynamic_cast<Epetra_Vector>(g_mp->getBlockVector()));
    }

    // dg/dp
    for (int j=0; j<num_p; j++) {
      if (!outArgs.supports(OUT_ARG_DgDp_mp, ii, j).none()) {
	MPDerivative dgdp_mp = outArgs.get_DgDp_mp(ii,j);
	Teuchos::RCP<Stokhos::ProductEpetraMultiVector> dgdp_mp_mv = 
	  dgdp_mp.getMultiVector();
	Teuchos::RCP<Epetra_Operator> dgdp_mp_op =
	  dgdp_mp.getLinearOp();
	if (dgdp_mp_mv != Teuchos::null) {
	  me_outargs.set_DgDp(
	    i, j, Derivative(dgdp_mp_mv->getBlockMultiVector(),
			     dgdp_mp.getMultiVectorOrientation()));
	}
	else if (dgdp_mp_op != Teuchos::null) {
	  me_outargs.set_DgDp(i, j, Derivative(dgdp_mp_op));
	}
      }
    }

  }

  // Compute the functions
  me->evalModel(me_inargs, me_outargs);

}
Beispiel #3
0
void Piro::Epetra::MatrixFreeDecorator::evalModel( const InArgs& inArgs,
                                     const OutArgs& outArgs ) const
{
  using Teuchos::RCP;
  using Teuchos::rcp;

  RCP<Epetra_Operator> W_out = outArgs.get_W();

  if (W_out == Teuchos::null) {
    // Just pass through as is: nothing to Decorate
    model->evalModel(inArgs, outArgs);
  }
  else {

    RCP<Piro::Epetra::MatrixFreeOperator> W_mfo =
      Teuchos::rcp_dynamic_cast<Piro::Epetra::MatrixFreeOperator>(W_out);

    TEUCHOS_TEST_FOR_EXCEPTION(W_mfo==Teuchos::null, std::logic_error, 
      "Epetra_Operator sent as W to Piro::Epetra::MatrixFreeDecorator\n" 
      "be of type Piro::Epetra::MatrixFreeOperator");
   
    // Do base case for MatrixFree: set f instead of W
    OutArgs modelOutArgs(outArgs);
    InArgs modelInArgs(inArgs);

    // Store f_out in case it was also requested
    RCP<Epetra_Vector> f_out = outArgs.get_f();

    modelOutArgs.set_f(fBase);
    modelOutArgs.set_W(Teuchos::null);

    //Evaluate the underlying model
    model->evalModel(modelInArgs, modelOutArgs);

    // If f_out was requested, return it.
    if (f_out != Teuchos::null) *f_out = *fBase;

    // Save unperturbed solution (deep copy inArgs, shallow f)
    InArgs clonedInArgs = inArgs;
    for (int l = 0; l < inArgs.Np(); ++l) {
      const RCP<const Epetra_Vector> p_l = inArgs.get_p(l);
      if (nonnull(p_l))
        clonedInArgs.set_p(l, Teuchos::rcp(new Epetra_Vector(*p_l)));
    }
    clonedInArgs.set_x(Teuchos::rcp(new Epetra_Vector(*inArgs.get_x())));

    bool haveXdot = false;
    if (inArgs.supports(IN_ARG_x_dot)) {
      RCP<const Epetra_Vector> xdot = inArgs.get_x_dot();
      if (nonnull(xdot)) {
        clonedInArgs.set_x_dot(Teuchos::rcp(new Epetra_Vector(*inArgs.get_x_dot())));
        haveXdot = true;
      }
    }
    W_mfo->setBase(clonedInArgs, fBase, haveXdot);
  }
}
void
twoD_diffusion_ME::
evalModel(const InArgs& inArgs, const OutArgs& outArgs) const
{

  //
  // Determinisic calculation
  //

  // Solution vector
  Teuchos::RCP<const Epetra_Vector> det_x = inArgs.get_x();

  // Parameters
  Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(0);
  if (p == Teuchos::null)
    p = p_init;

  Teuchos::RCP<Epetra_Vector> f = outArgs.get_f();
  Teuchos::RCP<Epetra_Operator> W = outArgs.get_W();
  Teuchos::RCP<Epetra_Operator> WPrec = outArgs.get_WPrec();
  if (f != Teuchos::null || W != Teuchos::null || WPrec != Teuchos::null) {
    if (basis != Teuchos::null) {
      for (int i=0; i<point.size(); i++)
        point[i] = (*p)[i];
      basis->evaluateBases(point, basis_vals);
      A->PutScalar(0.0);
      for (int k=0;k<A_k.size();k++)
        EpetraExt::MatrixMatrix::Add((*A_k[k]), false, basis_vals[k], *A, 1.0);
    }
    else {
      *A = *(A_k[0]);
      for (int k=1;k<A_k.size();k++)
        EpetraExt::MatrixMatrix::Add((*A_k[k]), false, (*p)[k-1], *A, 1.0);
    }
    A->FillComplete();
    A->OptimizeStorage();
  }

  // Residual
  if (f != Teuchos::null) {
    Teuchos::RCP<Epetra_Vector> kx = Teuchos::rcp(new Epetra_Vector(*x_map));
    A->Apply(*det_x,*kx);
    f->Update(1.0,*kx,-1.0, *b, 0.0);
  }

  // Jacobian
  if (W != Teuchos::null) {
    Teuchos::RCP<Epetra_CrsMatrix> jac =
      Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W, true);
    *jac = *A;
    jac->FillComplete();
    jac->OptimizeStorage();
  }

  // Preconditioner
  if (WPrec != Teuchos::null)
    precFactory->recompute(A, WPrec);

  // Responses (mean value)
  Teuchos::RCP<Epetra_Vector> g = outArgs.get_g(0);
  if (g != Teuchos::null) {
    (det_x->MeanValue(&(*g)[0]));
    (*g)[0] *= double(det_x->GlobalLength()) / double(mesh.size());
  }

  //
  // Stochastic Galerkin calculation
  //

  // Stochastic solution vector
  InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();

  // Stochastic parameters
  InArgs::sg_const_vector_t p_sg = inArgs.get_p_sg(0);

  // Stochastic residual
  OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
  if (f_sg != Teuchos::null) {

    // Get stochastic expansion data
    Teuchos::RCP<Stokhos::OrthogPolyExpansion<int,double> > expn =
      inArgs.get_sg_expansion();
    typedef Stokhos::Sparse3Tensor<int,double> Cijk_type;
    Teuchos::RCP<const Cijk_type> Cijk = expn->getTripleProduct();
    const Teuchos::Array<double>& norms = basis->norm_squared();

    if (sg_kx_vec_all.size() != basis->size()) {
      sg_kx_vec_all.resize(basis->size());
      for (int i=0;i<basis->size();i++) {
        sg_kx_vec_all[i] = Teuchos::rcp(new Epetra_Vector(*x_map));
      }
    }
    f_sg->init(0.0);

    Cijk_type::k_iterator k_begin = Cijk->k_begin();
    Cijk_type::k_iterator k_end = Cijk->k_end();
    for (Cijk_type::k_iterator k_it=k_begin; k_it!=k_end; ++k_it) {
      int k = Stokhos::index(k_it);
      for (Cijk_type::kj_iterator j_it = Cijk->j_begin(k_it);
           j_it != Cijk->j_end(k_it); ++j_it) {
        int j = Stokhos::index(j_it);
        A_k[k]->Apply((*x_sg)[j],*(sg_kx_vec_all[j]));
      }
      for (Cijk_type::kj_iterator j_it = Cijk->j_begin(k_it);
           j_it != Cijk->j_end(k_it); ++j_it) {
        int j = Stokhos::index(j_it);
        for (Cijk_type::kji_iterator i_it = Cijk->i_begin(j_it);
             i_it != Cijk->i_end(j_it); ++i_it) {
          int i = Stokhos::index(i_it);
          double c = Stokhos::value(i_it);  // C(i,j,k)
          (*f_sg)[i].Update(1.0*c/norms[i],*(sg_kx_vec_all[j]),1.0);
        }
      }
    } //End
    (*f_sg)[0].Update(-1.0,*b,1.0);
  }

  // Stochastic Jacobian
  OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
  if (W_sg != Teuchos::null) {
    for (int i=0; i<W_sg->size(); i++) {
      Teuchos::RCP<Epetra_CrsMatrix> jac =
        Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(i), true);
      *jac = *A_k[i];
      jac->FillComplete();
      jac->OptimizeStorage();
    }
  }

  // Stochastic responses
  Teuchos::RCP< Stokhos::EpetraVectorOrthogPoly > g_sg =
    outArgs.get_g_sg(0);
  if (g_sg != Teuchos::null) {
    int sz = x_sg->size();
    for (int i=0; i<sz; i++) {
      (*x_sg)[i].MeanValue(&(*g_sg)[i][0]);
      (*g_sg)[i][0] *= double((*x_sg)[i].GlobalLength()) / double(mesh.size());
    }
  }

  //
  // Multi-point calculation
  //

  // Stochastic solution vector
  mp_const_vector_t x_mp = inArgs.get_x_mp();

  // Stochastic parameters
  mp_const_vector_t p_mp = inArgs.get_p_mp(0);

  // Stochastic residual
  mp_vector_t f_mp = outArgs.get_f_mp();
  mp_operator_t W_mp = outArgs.get_W_mp();
  if (f_mp != Teuchos::null || W_mp != Teuchos::null) {
    int num_mp = x_mp->size();
    for (int i=0; i<num_mp; i++) {
      // Compute operator
      if (basis != Teuchos::null) {
        for (int k=0; k<point.size(); k++)
          point[k] = (*p_mp)[i][k];
        basis->evaluateBases(point, basis_vals);
        A->PutScalar(0.0);
        for (int k=0;k<A_k.size();k++)
          EpetraExt::MatrixMatrix::Add((*A_k[k]), false, basis_vals[k], *A,
                                       1.0);
      }
      else {
        *A = *(A_k[0]);
        for (int k=1;k<A_k.size();k++)
          EpetraExt::MatrixMatrix::Add((*A_k[k]), false, (*p_mp)[i][k-1], *A,
                                       1.0);
      }

      A->FillComplete();
      A->OptimizeStorage();

      // Compute residual
      if (f_mp != Teuchos::null) {
        A->Apply((*x_mp)[i], (*f_mp)[i]);
        (*f_mp)[i].Update(-1.0, *b, 1.0);
      }

      // Copy operator
      if (W_mp != Teuchos::null) {
        Teuchos::RCP<Epetra_CrsMatrix> jac =
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_mp->getCoeffPtr(i),
                                                      true);
        *jac = *A;
        jac->FillComplete();
        jac->OptimizeStorage();
      }
    }
  }

  // Multipoint responses
  mp_vector_t g_mp = outArgs.get_g_mp(0);
  if (g_mp != Teuchos::null) {
    int sz = x_mp->size();
    for (int i=0; i<sz; i++) {
      (*x_mp)[i].MeanValue(&(*g_mp)[i][0]);
      (*g_mp)[i][0] *= double((*x_mp)[i].GlobalLength()) / double(mesh.size());
    }
  }

}
void MockModelEval_A::evalModel( const InArgs& inArgs,
                              const OutArgs& outArgs ) const
{

  // Parse InArgs
  RCP<const Epetra_Vector> p_in = inArgs.get_p(0);
  if (!p_in.get()) cout << "ERROR: MockModelEval_A requires p as inargs" << endl;
  //int numParameters = p_in->GlobalLength();

  RCP<const Epetra_Vector> x_in = inArgs.get_x();
  if (!x_in.get()) cout << "ERROR: MockModelEval_A requires x as inargs" << endl;
  int vecLength = x_in->GlobalLength();
  int myVecLength = x_in->MyLength();

  // Parse OutArgs

  RCP<Epetra_Vector> f_out = outArgs.get_f(); 
  RCP<Epetra_Vector> g_out = outArgs.get_g(0); 
  Teuchos::RCP<Epetra_Operator> W_out = outArgs.get_W();
  Teuchos::RCP<Epetra_MultiVector> dfdp_out;
  if (outArgs.Np() > 0)
    dfdp_out = outArgs.get_DfDp(0).getMultiVector();
  RCP<Epetra_MultiVector> dgdp_out;
  dgdp_out = outArgs.get_DgDp(0,0).getMultiVector();
  RCP<Epetra_MultiVector> dgdx_out;
  dgdx_out = outArgs.get_DgDx(0).getMultiVector();

  if (f_out != Teuchos::null) {
    for (int i=0; i<myVecLength; i++) {
      int gid = x_in->Map().GID(i);
      if (gid==0) // x_0^2 = p_0
       (*f_out)[i] = (*x_in)[i] * (*x_in)[i] -  (*p_in)[i];
      else // x^2 = (i+p_1)^2
       (*f_out)[i] = (*x_in)[i] * (*x_in)[i] - (gid + (*p_in)[1])*(gid + (*p_in)[1]);
    }
  }
  if (W_out != Teuchos::null) {
    Teuchos::RCP<Epetra_CrsMatrix> W_out_crs =
      Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);
    W_out_crs->PutScalar(0.0);

    double diag=0.0;
    for (int i=0; i<myVecLength; i++) {
      diag = 2.0 * (*x_in)[i];
      W_out_crs->ReplaceMyValues(i, 1, &diag, &i);
    }
  }

  if (dfdp_out != Teuchos::null) {
    dfdp_out->PutScalar(0.0);
    for (int i=0; i<myVecLength; i++) {
      int gid = x_in->Map().GID(i);
      if   (gid==0) (*dfdp_out)[0][i] = -1.0;
      else          (*dfdp_out)[1][i] =  -2.0* (gid + (*p_in)[1]);
    }
  }

  // ObjFn = 0.5*(Sum(x)-Sum(p)-12)^2 + 0.5*(p0-1)^2:  min at 1,3

  double term1, term2;
  x_in->MeanValue(&term1); 
  term1 =  vecLength * term1 - ((*p_in)[0] + (*p_in)[1]) - 12.0;
  term2 = (*p_in)[0] - 1.0;
  
  if (!is_null(g_out)) {
    (*g_out)[0] = 0.5*term1*term1 + 0.5*term2*term2;
  }

  if (dgdx_out != Teuchos::null) {
     dgdx_out->PutScalar(term1);
   }
  if (dgdp_out != Teuchos::null) {
     dgdp_out->PutScalar(0.0);
     (*dgdp_out)[0][0] = -term1 + term2;
     (*dgdp_out)[0][1] = -term1;
   }

  // Modify for time dependent (implicit timeintegration or eigensolves
  // Check if time dependent
  RCP<const Epetra_Vector> x_dot = inArgs.get_x_dot();

  if (x_dot.get()) {
    double alpha =  inArgs.get_alpha();
    double beta  =  inArgs.get_beta();
    if (alpha==0.0 && beta==0.0) {
      cout << "MockModelEval Warning: alpha=beta=0 -- setting beta=1" << endl;
      beta = 1.0;
    }

    if (f_out != Teuchos::null) {
      for (int i=0; i<myVecLength; i++) {
         (*f_out)[i] = -alpha*(*x_dot)[i] + beta * (*f_out)[i];
      }
    }
    if (dfdp_out != Teuchos::null) {
      dfdp_out->Scale(beta);
    }
    if (W_out != Teuchos::null) {
      Teuchos::RCP<Epetra_CrsMatrix> W_out_crs =
        Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);
      W_out_crs->Scale(beta);

      double diag = -alpha;
      for (int i=0; i<myVecLength; i++) {
        W_out_crs->SumIntoMyValues(i, 1, &diag, &i);
      }
   cout << " W_crs  = " << *W_out_crs << endl;
    }
  } 
} 
void
Albany::ModelEvaluator::evalModel(const InArgs& inArgs,
                                 const OutArgs& outArgs) const
{
  Teuchos::TimeMonitor Timer(*timer); //start timer
  //
  // Get the input arguments
  //
  Teuchos::RCP<const Epetra_Vector> x = inArgs.get_x();
  Teuchos::RCP<const Epetra_Vector> x_dot;
  Teuchos::RCP<const Epetra_Vector> x_dotdot;
  double alpha     = 0.0;
  double omega     = 0.0;
  double beta      = 1.0;
  double curr_time = 0.0;
  x_dot = inArgs.get_x_dot();
  x_dotdot = inArgs.get_x_dotdot();
  if (x_dot != Teuchos::null || x_dotdot != Teuchos::null) {
    alpha = inArgs.get_alpha();
    omega = inArgs.get_omega();
    beta = inArgs.get_beta();
    curr_time  = inArgs.get_t();
  }
  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i);
    if (p != Teuchos::null) {
      for (unsigned int j=0; j<sacado_param_vec[i].size(); j++)
        sacado_param_vec[i][j].baseValue = (*p)[j];
    }
  }
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i+num_param_vecs);
    if (p != Teuchos::null) {
      *(distParamLib->get(dist_param_names[i])->vector()) = *p;
    }
  }

  //
  // Get the output arguments
  //
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> f_out = outArgs.get_f();
  Teuchos::RCP<Epetra_Operator> W_out = outArgs.get_W();

  // Cast W to a CrsMatrix, throw an exception if this fails
  Teuchos::RCP<Epetra_CrsMatrix> W_out_crs;

  if (W_out != Teuchos::null)
    W_out_crs = Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);

int test_var = 0;
if(test_var != 0){
std::cout << "The current solution length is: " << x->MyLength() << std::endl;
x->Print(std::cout);

}

  // Get preconditioner operator, if requested
  Teuchos::RCP<Epetra_Operator> WPrec_out;
  if (outArgs.supports(OUT_ARG_WPrec)) WPrec_out = outArgs.get_WPrec();

  //
  // Compute the functions
  //
  bool f_already_computed = false;

  // W matrix
  if (W_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(),*x,
                               sacado_param_vec, f_out.get(), *W_out_crs);
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current Jacobian length is: " << W_out_crs->NumGlobalRows() << std::endl;
W_out_crs->Print(std::cout);
}
  }

  if (WPrec_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(), *x,
                               sacado_param_vec, f_out.get(), *Extra_W_crs);
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current preconditioner length is: " << Extra_W_crs->NumGlobalRows() << std::endl;
Extra_W_crs->Print(std::cout);
}

    app->computeGlobalPreconditioner(Extra_W_crs, WPrec_out);
  }

  // scalar df/dp
  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<Epetra_MultiVector> dfdp_out =
      outArgs.get_DfDp(i).getMultiVector();
    if (dfdp_out != Teuchos::null) {
      Teuchos::Array<int> p_indexes =
        outArgs.get_DfDp(i).getDerivativeMultiVector().getParamIndexes();
      Teuchos::RCP<ParamVec> p_vec;
      if (p_indexes.size() == 0)
        p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
      else {
        p_vec = Teuchos::rcp(new ParamVec);
        for (int j=0; j<p_indexes.size(); j++)
          p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                          sacado_param_vec[i][p_indexes[j]].baseValue);
      }

      app->computeGlobalTangent(0.0, 0.0, 0.0, curr_time, false, x_dot.get(), x_dotdot.get(), *x,
                                sacado_param_vec, p_vec.get(),
                                NULL, NULL, NULL, NULL, f_out.get(), NULL,
                                dfdp_out.get());

      f_already_computed=true;
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }

  // distributed df/dp
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<Epetra_Operator> dfdp_out =
      outArgs.get_DfDp(i+num_param_vecs).getLinearOp();
    if (dfdp_out != Teuchos::null) {
      Teuchos::RCP<DistributedParameterDerivativeOp> dfdp_op =
        Teuchos::rcp_dynamic_cast<DistributedParameterDerivativeOp>(dfdp_out);
      dfdp_op->set(curr_time, x_dot, x_dotdot, x,
                   Teuchos::rcp(&sacado_param_vec,false));
    }
  }

  // f
  if (app->is_adjoint) {
    Derivative f_deriv(f_out, DERIV_TRANS_MV_BY_ROW);
    int response_index = 0; // need to add capability for sending this in
    app->evaluateResponseDerivative(response_index, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                    sacado_param_vec, NULL,
                                    NULL, f_deriv, Derivative(), Derivative(), Derivative());
  }
  else {
    if (f_out != Teuchos::null && !f_already_computed) {
      app->computeGlobalResidual(curr_time, x_dot.get(), x_dotdot.get(), *x,
                                 sacado_param_vec, *f_out);
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }

  // Response functions
  for (int i=0; i<outArgs.Ng(); i++) {
    Teuchos::RCP<Epetra_Vector> g_out = outArgs.get_g(i);
    bool g_computed = false;

    Derivative dgdx_out = outArgs.get_DgDx(i);
    Derivative dgdxdot_out = outArgs.get_DgDx_dot(i);
    Derivative dgdxdotdot_out = outArgs.get_DgDx_dotdot(i);

    // dg/dx, dg/dxdot
    if (!dgdx_out.isEmpty() || !dgdxdot_out.isEmpty() || !dgdxdotdot_out.isEmpty() ) {
      app->evaluateResponseDerivative(i, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                      sacado_param_vec, NULL,
                                      g_out.get(), dgdx_out,
                                      dgdxdot_out, dgdxdotdot_out, Derivative());
      g_computed = true;
    }

    // dg/dp
    for (int j=0; j<num_param_vecs; j++) {
      Teuchos::RCP<Epetra_MultiVector> dgdp_out =
        outArgs.get_DgDp(i,j).getMultiVector();
      if (dgdp_out != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DgDp(i,j).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int k=0; k<p_indexes.size(); k++)
            p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                            sacado_param_vec[j][p_indexes[k]].baseValue);
        }
        app->evaluateResponseTangent(i, alpha, beta, omega, curr_time, false,
                                     x_dot.get(), x_dotdot.get(), *x,
                                     sacado_param_vec, p_vec.get(),
                                     NULL, NULL, NULL, NULL, g_out.get(), NULL,
                                     dgdp_out.get());
        g_computed = true;
      }
    }

    // Need to handle dg/dp for distributed p

    if (g_out != Teuchos::null && !g_computed)
      app->evaluateResponse(i, curr_time, x_dot.get(), x_dotdot.get(), *x, sacado_param_vec,
                            *g_out);
  }

  //
  // Stochastic Galerkin
  //
#ifdef ALBANY_SG_MP
  InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();
  if (x_sg != Teuchos::null) {
    app->init_sg(inArgs.get_sg_basis(),
                 inArgs.get_sg_quadrature(),
                 inArgs.get_sg_expansion(),
                 x_sg->productComm());
    InArgs::sg_const_vector_t x_dot_sg  = inArgs.get_x_dot_sg();
    InArgs::sg_const_vector_t x_dotdot_sg  = inArgs.get_x_dotdot_sg();
    if (x_dot_sg != Teuchos::null || x_dotdot_sg != Teuchos::null) {
      alpha = inArgs.get_alpha();
      omega = inArgs.get_omega();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }

    InArgs::sg_const_vector_t epetra_p_sg = inArgs.get_p_sg(0);
    Teuchos::Array<int> p_sg_index;
    for (int i=0; i<num_param_vecs; i++) {
      InArgs::sg_const_vector_t p_sg = inArgs.get_p_sg(i);
      if (p_sg != Teuchos::null) {
        p_sg_index.push_back(i);
        for (int j=0; j<p_sg_vals[i].size(); j++) {
          int num_sg_blocks = p_sg->size();
          p_sg_vals[i][j].reset(app->getStochasticExpansion(), num_sg_blocks);
          p_sg_vals[i][j].copyForWrite();
          for (int l=0; l<num_sg_blocks; l++) {
            p_sg_vals[i][j].fastAccessCoeff(l) = (*p_sg)[l][j];
          }
        }
      }
    }

    OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
    OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
    bool f_sg_computed = false;

    // W_sg
    if (W_sg != Teuchos::null) {
      Stokhos::VectorOrthogPoly<Epetra_CrsMatrix> W_sg_crs(W_sg->basis(),
                                                           W_sg->map());
      for (int i=0; i<W_sg->size(); i++)
        W_sg_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(i)));
      app->computeGlobalSGJacobian(alpha, beta, omega, curr_time,
                                   x_dot_sg.get(),  x_dotdot_sg.get(), *x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   f_sg.get(), W_sg_crs);
      f_sg_computed = true;
    }

    // df/dp_sg
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dfdp_sg
        = outArgs.get_DfDp_sg(i).getMultiVector();
      if (dfdp_sg != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_sg(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalSGTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                    sacado_param_vec, p_sg_index, p_sg_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_sg.get(), NULL, dfdp_sg.get());

        f_sg_computed = true;
      }
    }

    if (f_sg != Teuchos::null && !f_sg_computed)
      app->computeGlobalSGResidual(curr_time, x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   *f_sg);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      OutArgs::sg_vector_t g_sg = outArgs.get_g_sg(i);
      bool g_sg_computed = false;

      SGDerivative dgdx_sg = outArgs.get_DgDx_sg(i);
      SGDerivative dgdxdot_sg = outArgs.get_DgDx_dot_sg(i);
      SGDerivative dgdxdotdot_sg = outArgs.get_DgDx_dotdot_sg(i);

      // dg/dx, dg/dxdot
      if (!dgdx_sg.isEmpty() || !dgdxdot_sg.isEmpty() || !dgdxdotdot_sg.isEmpty()) {
        app->evaluateSGResponseDerivative(
            i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
            sacado_param_vec, p_sg_index, p_sg_vals,
            NULL, g_sg.get(), dgdx_sg,
            dgdxdot_sg, dgdxdotdot_sg, SGDerivative());
        g_sg_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dgdp_sg =
          outArgs.get_DgDp_sg(i,j).getMultiVector();
        if (dgdp_sg != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_sg(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateSGResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                         sacado_param_vec, p_sg_index,
                                         p_sg_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_sg.get(),
                                         NULL, dgdp_sg.get());
          g_sg_computed = true;

        }
      }

      if (g_sg != Teuchos::null && !g_sg_computed)
        app->evaluateSGResponse(i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                sacado_param_vec, p_sg_index, p_sg_vals,
                                *g_sg);
    }
  }

  //
  // Multi-point evaluation
  //
  mp_const_vector_t x_mp = inArgs.get_x_mp();
  if (x_mp != Teuchos::null) {
    mp_const_vector_t x_dot_mp  = inArgs.get_x_dot_mp();
    mp_const_vector_t x_dotdot_mp  = inArgs.get_x_dotdot_mp();
    if (x_dot_mp != Teuchos::null || x_dotdot_mp != Teuchos::null) {
      alpha = inArgs.get_alpha();
      omega = inArgs.get_omega();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }

    Teuchos::Array<int> p_mp_index;
    for (int i=0; i<num_param_vecs; i++) {
      mp_const_vector_t p_mp = inArgs.get_p_mp(i);
      if (p_mp != Teuchos::null) {
        p_mp_index.push_back(i);
        for (int j=0; j<p_mp_vals[i].size(); j++) {
          int num_mp_blocks = p_mp->size();
          p_mp_vals[i][j].reset(num_mp_blocks);
          p_mp_vals[i][j].copyForWrite();
          for (int l=0; l<num_mp_blocks; l++) {
            p_mp_vals[i][j].fastAccessCoeff(l) = (*p_mp)[l][j];
          }
        }
      }
    }

    mp_vector_t f_mp = outArgs.get_f_mp();
    mp_operator_t W_mp = outArgs.get_W_mp();
    bool f_mp_computed = false;

    // W_mp
    if (W_mp != Teuchos::null) {
      Stokhos::ProductContainer<Epetra_CrsMatrix> W_mp_crs(W_mp->map());
      for (int i=0; i<W_mp->size(); i++)
        W_mp_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_mp->getCoeffPtr(i)));
      app->computeGlobalMPJacobian(alpha, beta, omega, curr_time,
                                   x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   f_mp.get(), W_mp_crs);
      f_mp_computed = true;
    }

    // df/dp_mp
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dfdp_mp
        = outArgs.get_DfDp_mp(i).getMultiVector();
      if (dfdp_mp != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_mp(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalMPTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                    sacado_param_vec, p_mp_index, p_mp_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_mp.get(), NULL, dfdp_mp.get());

        f_mp_computed = true;
      }
    }

    if (f_mp != Teuchos::null && !f_mp_computed)
      app->computeGlobalMPResidual(curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   *f_mp);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      mp_vector_t g_mp = outArgs.get_g_mp(i);
      bool g_mp_computed = false;

      MPDerivative dgdx_mp = outArgs.get_DgDx_mp(i);
      MPDerivative dgdxdot_mp = outArgs.get_DgDx_dot_mp(i);
      MPDerivative dgdxdotdot_mp = outArgs.get_DgDx_dotdot_mp(i);

      // dg/dx, dg/dxdot
      if (!dgdx_mp.isEmpty() || !dgdxdot_mp.isEmpty() || !dgdxdotdot_mp.isEmpty() ) {
        app->evaluateMPResponseDerivative(
            i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
            sacado_param_vec, p_mp_index, p_mp_vals,
            NULL, g_mp.get(), dgdx_mp,
            dgdxdot_mp, dgdxdotdot_mp, MPDerivative());
        g_mp_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dgdp_mp =
          outArgs.get_DgDp_mp(i,j).getMultiVector();
        if (dgdp_mp != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_mp(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateMPResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                         sacado_param_vec, p_mp_index,
                                         p_mp_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_mp.get(),
                                         NULL, dgdp_mp.get());
          g_mp_computed = true;
        }
      }

      if (g_mp != Teuchos::null && !g_mp_computed)
        app->evaluateMPResponse(i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                sacado_param_vec, p_mp_index, p_mp_vals,
                                *g_mp);
    }
  }
#endif //ALBANY_SG_MP
}
void
Albany::ModelEvaluator::evalModel(const InArgs& inArgs,
                                 const OutArgs& outArgs) const
{
  Teuchos::TimeMonitor Timer(*timer); //start timer
  //
  // Get the input arguments
  //
  Teuchos::RCP<const Epetra_Vector> x = inArgs.get_x();
  Teuchos::RCP<const Epetra_Vector> x_dot;
  Teuchos::RCP<const Epetra_Vector> x_dotdot;

  //create comm and node objects for Epetra -> Tpetra conversions
  Teuchos::RCP<const Teuchos::Comm<int> > commT = app->getComm();
  Teuchos::RCP<Epetra_Comm> comm = Albany::createEpetraCommFromTeuchosComm(commT);
  //Create Tpetra copy of x, call it xT
  Teuchos::RCP<const Tpetra_Vector> xT;
  if (x != Teuchos::null)
    xT  = Petra::EpetraVector_To_TpetraVectorConst(*x, commT);

  double alpha     = 0.0;
  double omega     = 0.0;
  double beta      = 1.0;
  double curr_time = 0.0;

  if(num_time_deriv > 0)
    x_dot = inArgs.get_x_dot();
  if(num_time_deriv > 1)
    x_dotdot = inArgs.get_x_dotdot();

  //Declare and create Tpetra copy of x_dot, call it x_dotT
  Teuchos::RCP<const Tpetra_Vector> x_dotT;
  if (Teuchos::nonnull(x_dot))
    x_dotT = Petra::EpetraVector_To_TpetraVectorConst(*x_dot, commT);

  //Declare and create Tpetra copy of x_dotdot, call it x_dotdotT
  Teuchos::RCP<const Tpetra_Vector> x_dotdotT;
  if (Teuchos::nonnull(x_dotdot))
    x_dotdotT = Petra::EpetraVector_To_TpetraVectorConst(*x_dotdot, commT);

  if (Teuchos::nonnull(x_dot)){
    alpha = inArgs.get_alpha();
    beta = inArgs.get_beta();
    curr_time  = inArgs.get_t();
  }
  if (Teuchos::nonnull(x_dotdot)) {
    omega = inArgs.get_omega();
  }

  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i);
    if (p != Teuchos::null) {
      for (unsigned int j=0; j<sacado_param_vec[i].size(); j++) {
        sacado_param_vec[i][j].baseValue = (*p)[j];
      }
    }
  }
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<const Epetra_Vector> p = inArgs.get_p(i+num_param_vecs);
    //create Tpetra copy of p
    Teuchos::RCP<const Tpetra_Vector> pT;
    if (p != Teuchos::null) {
      pT = Petra::EpetraVector_To_TpetraVectorConst(*p, commT);
      //*(distParamLib->get(dist_param_names[i])->vector()) = *p;
      *(distParamLib->get(dist_param_names[i])->vector()) = *pT;
    }
  }

  //
  // Get the output arguments
  //
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> f_out = outArgs.get_f();
  Teuchos::RCP<Epetra_Operator> W_out = outArgs.get_W();

  // Cast W to a CrsMatrix, throw an exception if this fails
  Teuchos::RCP<Epetra_CrsMatrix> W_out_crs;
#ifdef WRITE_MASS_MATRIX_TO_MM_FILE
  //IK, 7/15/14: adding object to hold mass matrix to be written to matrix market file
  Teuchos::RCP<Epetra_CrsMatrix> Mass;
  //IK, 7/15/14: needed for writing mass matrix out to matrix market file
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> ftmp = outArgs.get_f();
#endif

  if (W_out != Teuchos::null) {
    W_out_crs = Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);
#ifdef WRITE_MASS_MATRIX_TO_MM_FILE
    //IK, 7/15/14: adding object to hold mass matrix to be written to matrix market file
    Mass = Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_out, true);
#endif
  }


int test_var = 0;
if(test_var != 0){
std::cout << "The current solution length is: " << x->MyLength() << std::endl;
x->Print(std::cout);

}

  // Get preconditioner operator, if requested
  Teuchos::RCP<Epetra_Operator> WPrec_out;
  if (outArgs.supports(OUT_ARG_WPrec)) WPrec_out = outArgs.get_WPrec();

  //
  // Compute the functions
  //
  bool f_already_computed = false;

  // W matrix
  if (W_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(),*x,
                               sacado_param_vec, f_out.get(), *W_out_crs);
#ifdef WRITE_MASS_MATRIX_TO_MM_FILE
    //IK, 7/15/14: write mass matrix to matrix market file
    //Warning: to read this in to MATLAB correctly, code must be run in serial.
    //Otherwise Mass will have a distributed Map which would also need to be read in to MATLAB for proper
    //reading in of Mass.
    app->computeGlobalJacobian(1.0, 0.0, 0.0, curr_time, x_dot.get(), x_dotdot.get(), *x,
                               sacado_param_vec, ftmp.get(), *Mass);
    EpetraExt::RowMatrixToMatrixMarketFile("mass.mm", *Mass);
    EpetraExt::BlockMapToMatrixMarketFile("rowmap.mm", Mass->RowMap());
    EpetraExt::BlockMapToMatrixMarketFile("colmap.mm", Mass->ColMap());
    Teuchos::RCP<Teuchos::FancyOStream> out = Teuchos::VerboseObjectBase::getDefaultOStream();
#endif
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current Jacobian length is: " << W_out_crs->NumGlobalRows() << std::endl;
W_out_crs->Print(std::cout);
}
  }

  if (WPrec_out != Teuchos::null) {
    app->computeGlobalJacobian(alpha, beta, omega, curr_time, x_dot.get(), x_dotdot.get(), *x,
                               sacado_param_vec, f_out.get(), *Extra_W_crs);
    f_already_computed=true;
if(test_var != 0){
//std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
//f_out->Print(std::cout);
std::cout << "The current preconditioner length is: " << Extra_W_crs->NumGlobalRows() << std::endl;
Extra_W_crs->Print(std::cout);
}

    app->computeGlobalPreconditioner(Extra_W_crs, WPrec_out);
  }

  // scalar df/dp
  for (int i=0; i<num_param_vecs; i++) {
    Teuchos::RCP<Epetra_MultiVector> dfdp_out =
      outArgs.get_DfDp(i).getMultiVector();
    if (dfdp_out != Teuchos::null) {
      Teuchos::Array<int> p_indexes =
        outArgs.get_DfDp(i).getDerivativeMultiVector().getParamIndexes();
      Teuchos::RCP<ParamVec> p_vec;
      if (p_indexes.size() == 0)
        p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
      else {
        p_vec = Teuchos::rcp(new ParamVec);
        for (int j=0; j<p_indexes.size(); j++)
          p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                          sacado_param_vec[i][p_indexes[j]].baseValue);
      }

      app->computeGlobalTangent(0.0, 0.0, 0.0, curr_time, false, x_dot.get(), x_dotdot.get(), *x,
                                sacado_param_vec, p_vec.get(),
                                NULL, NULL, NULL, NULL, f_out.get(), NULL,
                                dfdp_out.get());

      f_already_computed=true;
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }

  // distributed df/dp
  for (int i=0; i<num_dist_param_vecs; i++) {
    Teuchos::RCP<Epetra_Operator> dfdp_out =
      outArgs.get_DfDp(i+num_param_vecs).getLinearOp();
    if (dfdp_out != Teuchos::null) {
      Teuchos::RCP<DistributedParameterDerivativeOp> dfdp_op =
        Teuchos::rcp_dynamic_cast<DistributedParameterDerivativeOp>(dfdp_out);
      dfdp_op->set(curr_time, x_dotT, x_dotdotT, xT,
                   Teuchos::rcp(&sacado_param_vec,false));
    }
  }

  // f
  if (app->is_adjoint) {
    Derivative f_deriv(f_out, DERIV_TRANS_MV_BY_ROW);
    int response_index = 0; // need to add capability for sending this in
    app->evaluateResponseDerivative(response_index, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                    sacado_param_vec, NULL,
                                    NULL, f_deriv, Derivative(), Derivative(), Derivative());
  }
  else {
    if (f_out != Teuchos::null && !f_already_computed) {
      app->computeGlobalResidual(curr_time, x_dot.get(), x_dotdot.get(), *x,
                                  sacado_param_vec, *f_out);
if(test_var != 0){
std::cout << "The current rhs length is: " << f_out->MyLength() << std::endl;
f_out->Print(std::cout);
}
    }
  }


  // Response functions
  for (int i=0; i<outArgs.Ng(); i++) {
    //Set curr_time to final time at which response occurs.
    if(num_time_deriv > 0)
      curr_time  = inArgs.get_t();
    Teuchos::RCP<Epetra_Vector> g_out = outArgs.get_g(i);
    //Declare Tpetra_Vector copy of g_out
    Teuchos::RCP<Tpetra_Vector> g_outT;
    bool g_computed = false;

    Derivative dgdx_out = outArgs.get_DgDx(i);
    Derivative dgdxdot_out = outArgs.get_DgDx_dot(i);
    Derivative dgdxdotdot_out = outArgs.get_DgDx_dotdot(i);

    // dg/dx, dg/dxdot
    if (!dgdx_out.isEmpty() || !dgdxdot_out.isEmpty() || !dgdxdotdot_out.isEmpty() ) {
      app->evaluateResponseDerivative(i, curr_time, x_dot.get(), x_dotdot.get(), *x,
                                      sacado_param_vec, NULL,
                                      g_out.get(), dgdx_out,
                                      dgdxdot_out, dgdxdotdot_out, Derivative());
      g_computed = true;
    }

    // dg/dp
    for (int j=0; j<num_param_vecs; j++) {
      Teuchos::RCP<Epetra_MultiVector> dgdp_out =
        outArgs.get_DgDp(i,j).getMultiVector();
      //Declare Tpetra copy of dgdp_out
      Teuchos::RCP<Tpetra_MultiVector> dgdp_outT;
      if (dgdp_out != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DgDp(i,j).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int k=0; k<p_indexes.size(); k++)
            p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                            sacado_param_vec[j][p_indexes[k]].baseValue);
        }
        //create Tpetra copy of g_out, call it g_outT
        if (g_out != Teuchos::null)
           g_outT = Petra::EpetraVector_To_TpetraVectorNonConst(*g_out, commT);
        //create Tpetra copy of dgdp_out, call it dgdp_outT
        if (dgdp_out != Teuchos::null)
           dgdp_outT = Petra::EpetraMultiVector_To_TpetraMultiVector(*dgdp_out, commT);
	app->evaluateResponseTangentT(i, alpha, beta, omega, curr_time, false,
				     x_dotT.get(), x_dotdotT.get(), *xT,
				     sacado_param_vec, p_vec.get(),
				     NULL, NULL, NULL, NULL, g_outT.get(), NULL,
				     dgdp_outT.get());
        //convert g_outT to Epetra_Vector g_out
        if (g_out != Teuchos::null)
          Petra::TpetraVector_To_EpetraVector(g_outT, *g_out, comm);
        //convert dgdp_outT to Epetra_MultiVector dgdp_out
        if (dgdp_out != Teuchos::null)
          Petra::TpetraMultiVector_To_EpetraMultiVector(dgdp_outT, *dgdp_out, comm);
        g_computed = true;
      }
    }

    // Need to handle dg/dp for distributed p
    for(int j=0; j<num_dist_param_vecs; j++) {
      Derivative dgdp_out = outArgs.get_DgDp(i,j+num_param_vecs);
      if (!dgdp_out.isEmpty()) {
        dgdp_out.getMultiVector()->PutScalar(0.);
        app->evaluateResponseDistParamDeriv(i, curr_time, x_dot.get(), x_dotdot.get(), *x, sacado_param_vec, dist_param_names[j], dgdp_out.getMultiVector().get());
      }
    }

    if (g_out != Teuchos::null && !g_computed) {
      //create Tpetra copy of g_out, call it g_outT
      g_outT = Petra::EpetraVector_To_TpetraVectorNonConst(*g_out, commT);
      app->evaluateResponseT(i, curr_time, x_dotT.get(), x_dotdotT.get(), *xT, sacado_param_vec,
			    *g_outT);
      //convert g_outT to Epetra_Vector g_out
      Petra::TpetraVector_To_EpetraVector(g_outT, *g_out, comm);
    }
  }

  //
  // Stochastic Galerkin
  //
#ifdef ALBANY_SG
  InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();
  if (x_sg != Teuchos::null) {
    app->init_sg(inArgs.get_sg_basis(),
                 inArgs.get_sg_quadrature(),
                 inArgs.get_sg_expansion(),
                 x_sg->productComm());
    InArgs::sg_const_vector_t x_dot_sg  = Teuchos::null;
    InArgs::sg_const_vector_t x_dot_sg  = Teuchos::null;
    if(num_time_deriv > 0)
      x_dotdot_sg  = inArgs.get_x_dotdot_sg();
    if(num_time_deriv > 1)
      x_dotdot_sg  = inArgs.get_x_dotdot_sg();
    if (x_dot_sg != Teuchos::null || x_dotdot_sg != Teuchos::null) {
      alpha = inArgs.get_alpha();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }
    if (x_dotdot_sg != Teuchos::null) {
      omega = inArgs.get_omega();
    }

    InArgs::sg_const_vector_t epetra_p_sg = inArgs.get_p_sg(0);
    Teuchos::Array<int> p_sg_index;
    for (int i=0; i<num_param_vecs; i++) {
      InArgs::sg_const_vector_t p_sg = inArgs.get_p_sg(i);
      if (p_sg != Teuchos::null) {
        p_sg_index.push_back(i);
        for (int j=0; j<p_sg_vals[i].size(); j++) {
          int num_sg_blocks = p_sg->size();
          p_sg_vals[i][j].reset(app->getStochasticExpansion(), num_sg_blocks);
          p_sg_vals[i][j].copyForWrite();
          for (int l=0; l<num_sg_blocks; l++) {
            p_sg_vals[i][j].fastAccessCoeff(l) = (*p_sg)[l][j];
          }
        }
      }
    }

    OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
    OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
    bool f_sg_computed = false;

    // W_sg
    if (W_sg != Teuchos::null) {
      Stokhos::VectorOrthogPoly<Epetra_CrsMatrix> W_sg_crs(W_sg->basis(),
                                                           W_sg->map());
      for (int i=0; i<W_sg->size(); i++)
        W_sg_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(i)));
      app->computeGlobalSGJacobian(alpha, beta, omega, curr_time,
                                   x_dot_sg.get(),  x_dotdot_sg.get(), *x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   f_sg.get(), W_sg_crs);
      f_sg_computed = true;
    }

    // df/dp_sg
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dfdp_sg
        = outArgs.get_DfDp_sg(i).getMultiVector();
      if (dfdp_sg != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_sg(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalSGTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                    sacado_param_vec, p_sg_index, p_sg_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_sg.get(), NULL, dfdp_sg.get());

        f_sg_computed = true;
      }
    }

    if (f_sg != Teuchos::null && !f_sg_computed)
      app->computeGlobalSGResidual(curr_time, x_dot_sg.get(), x_dotdot_sg.get(),*x_sg,
                                   sacado_param_vec, p_sg_index, p_sg_vals,
                                   *f_sg);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      OutArgs::sg_vector_t g_sg = outArgs.get_g_sg(i);
      bool g_sg_computed = false;

      SGDerivative dgdx_sg = outArgs.get_DgDx_sg(i);
      SGDerivative dgdxdot_sg = outArgs.get_DgDx_dot_sg(i);
      SGDerivative dgdxdotdot_sg = outArgs.get_DgDx_dotdot_sg(i);

      // dg/dx, dg/dxdot
      if (!dgdx_sg.isEmpty() || !dgdxdot_sg.isEmpty() || !dgdxdotdot_sg.isEmpty()) {
        app->evaluateSGResponseDerivative(
            i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
            sacado_param_vec, p_sg_index, p_sg_vals,
            NULL, g_sg.get(), dgdx_sg,
            dgdxdot_sg, dgdxdotdot_sg, SGDerivative());
        g_sg_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::EpetraMultiVectorOrthogPoly > dgdp_sg =
          outArgs.get_DgDp_sg(i,j).getMultiVector();
        if (dgdp_sg != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_sg(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateSGResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                         sacado_param_vec, p_sg_index,
                                         p_sg_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_sg.get(),
                                         NULL, dgdp_sg.get());
          g_sg_computed = true;

        }
      }

      if (g_sg != Teuchos::null && !g_sg_computed)
        app->evaluateSGResponse(i, curr_time, x_dot_sg.get(), x_dotdot_sg.get(), *x_sg,
                                sacado_param_vec, p_sg_index, p_sg_vals,
                                *g_sg);
    }
  }
#endif
#ifdef ALBANY_ENSEMBLE

  //
  // Multi-point evaluation
  //
  mp_const_vector_t x_mp = inArgs.get_x_mp();
  if (x_mp != Teuchos::null) {
    mp_const_vector_t x_dot_mp  = Teuchos::null;
    mp_const_vector_t x_dotdot_mp  = Teuchos::null;
    if(num_time_deriv > 0)
      x_dot_mp  = inArgs.get_x_dot_mp();
    if(num_time_deriv > 1)
      x_dotdot_mp  = inArgs.get_x_dotdot_mp();
    if (x_dot_mp != Teuchos::null || x_dotdot_mp != Teuchos::null) {
      alpha = inArgs.get_alpha();
      //omega = inArgs.get_omega();
      beta = inArgs.get_beta();
      curr_time  = inArgs.get_t();
    }
    if (x_dotdot_mp != Teuchos::null) {
      omega = inArgs.get_omega();
    }

    Teuchos::Array<int> p_mp_index;
    for (int i=0; i<num_param_vecs; i++) {
      mp_const_vector_t p_mp = inArgs.get_p_mp(i);
      if (p_mp != Teuchos::null) {
        p_mp_index.push_back(i);
        for (int j=0; j<p_mp_vals[i].size(); j++) {
          int num_mp_blocks = p_mp->size();
          p_mp_vals[i][j].reset(num_mp_blocks);
          p_mp_vals[i][j].copyForWrite();
          for (int l=0; l<num_mp_blocks; l++) {
            p_mp_vals[i][j].fastAccessCoeff(l) = (*p_mp)[l][j];
          }
        }
      }
    }

    mp_vector_t f_mp = outArgs.get_f_mp();
    mp_operator_t W_mp = outArgs.get_W_mp();
    bool f_mp_computed = false;

    // W_mp
    if (W_mp != Teuchos::null) {
      Stokhos::ProductContainer<Epetra_CrsMatrix> W_mp_crs(W_mp->map());
      for (int i=0; i<W_mp->size(); i++)
        W_mp_crs.setCoeffPtr(
          i,
          Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_mp->getCoeffPtr(i)));
      app->computeGlobalMPJacobian(alpha, beta, omega, curr_time,
                                   x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   f_mp.get(), W_mp_crs);
      f_mp_computed = true;
    }

    // df/dp_mp
    for (int i=0; i<num_param_vecs; i++) {
      Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dfdp_mp
        = outArgs.get_DfDp_mp(i).getMultiVector();
      if (dfdp_mp != Teuchos::null) {
        Teuchos::Array<int> p_indexes =
          outArgs.get_DfDp_mp(i).getDerivativeMultiVector().getParamIndexes();
        Teuchos::RCP<ParamVec> p_vec;
        if (p_indexes.size() == 0)
          p_vec = Teuchos::rcp(&sacado_param_vec[i],false);
        else {
          p_vec = Teuchos::rcp(new ParamVec);
          for (int j=0; j<p_indexes.size(); j++)
            p_vec->addParam(sacado_param_vec[i][p_indexes[j]].family,
                            sacado_param_vec[i][p_indexes[j]].baseValue);
        }

        app->computeGlobalMPTangent(0.0, 0.0, 0.0, curr_time, false,
                                    x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                    sacado_param_vec, p_mp_index, p_mp_vals,
                                    p_vec.get(), NULL, NULL, NULL, NULL,
                                    f_mp.get(), NULL, dfdp_mp.get());

        f_mp_computed = true;
      }
    }

    if (f_mp != Teuchos::null && !f_mp_computed)
      app->computeGlobalMPResidual(curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                   sacado_param_vec, p_mp_index, p_mp_vals,
                                   *f_mp);

    // Response functions
    for (int i=0; i<outArgs.Ng(); i++) {
      mp_vector_t g_mp = outArgs.get_g_mp(i);
      bool g_mp_computed = false;

      MPDerivative dgdx_mp = outArgs.get_DgDx_mp(i);
      MPDerivative dgdxdot_mp = outArgs.get_DgDx_dot_mp(i);
      MPDerivative dgdxdotdot_mp = outArgs.get_DgDx_dotdot_mp(i);

      // dg/dx, dg/dxdot
      if (!dgdx_mp.isEmpty() || !dgdxdot_mp.isEmpty() || !dgdxdotdot_mp.isEmpty() ) {
        app->evaluateMPResponseDerivative(
            i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
            sacado_param_vec, p_mp_index, p_mp_vals,
            NULL, g_mp.get(), dgdx_mp,
            dgdxdot_mp, dgdxdotdot_mp, MPDerivative());
        g_mp_computed = true;
      }

      // dg/dp
      for (int j=0; j<num_param_vecs; j++) {
        Teuchos::RCP< Stokhos::ProductEpetraMultiVector > dgdp_mp =
          outArgs.get_DgDp_mp(i,j).getMultiVector();
        if (dgdp_mp != Teuchos::null) {
          Teuchos::Array<int> p_indexes =
            outArgs.get_DgDp_mp(i,j).getDerivativeMultiVector().getParamIndexes();
          Teuchos::RCP<ParamVec> p_vec;
          if (p_indexes.size() == 0)
            p_vec = Teuchos::rcp(&sacado_param_vec[j],false);
          else {
            p_vec = Teuchos::rcp(new ParamVec);
            for (int k=0; k<p_indexes.size(); k++)
              p_vec->addParam(sacado_param_vec[j][p_indexes[k]].family,
                              sacado_param_vec[j][p_indexes[k]].baseValue);
          }
          app->evaluateMPResponseTangent(i, alpha, beta, omega, curr_time, false,
                                         x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                         sacado_param_vec, p_mp_index,
                                         p_mp_vals, p_vec.get(),
                                         NULL, NULL, NULL, NULL, g_mp.get(),
                                         NULL, dgdp_mp.get());
          g_mp_computed = true;
        }
      }

      if (g_mp != Teuchos::null && !g_mp_computed)
        app->evaluateMPResponse(i, curr_time, x_dot_mp.get(), x_dotdot_mp.get(), *x_mp,
                                sacado_param_vec, p_mp_index, p_mp_vals,
                                *g_mp);
    }
  }
#endif
}
void 
Stokhos::SGQuadModelEvaluator::
evalModel(const InArgs& inArgs, const OutArgs& outArgs) const
{
  // Create underlying inargs
  InArgs me_inargs = me->createInArgs();
  if (me_inargs.supports(IN_ARG_x))
    me_inargs.set_x(inArgs.get_x());
  if (me_inargs.supports(IN_ARG_x_dot))
    me_inargs.set_x_dot(inArgs.get_x_dot());
  if (me_inargs.supports(IN_ARG_alpha))
    me_inargs.set_alpha(inArgs.get_alpha());
  if (me_inargs.supports(IN_ARG_beta))
    me_inargs.set_beta(inArgs.get_beta());
  if (me_inargs.supports(IN_ARG_t))
    me_inargs.set_t(inArgs.get_t());
  for (int i=0; i<num_p; i++)
    me_inargs.set_p(i, inArgs.get_p(i));

  // Create underlying outargs
  OutArgs me_outargs = me->createOutArgs();
  if (me_outargs.supports(OUT_ARG_f))
    me_outargs.set_f(outArgs.get_f());
  if (me_outargs.supports(OUT_ARG_W))
    me_outargs.set_W(outArgs.get_W());
  for (int j=0; j<num_p; j++)
    if (!outArgs.supports(OUT_ARG_DfDp, j).none())
      me_outargs.set_DfDp(j, outArgs.get_DfDp(j));
  for (int i=0; i<num_g; i++) {
    me_outargs.set_g(i, outArgs.get_g(i));
    if (!outArgs.supports(OUT_ARG_DgDx, i).none())
	me_outargs.set_DgDx(i, outArgs.get_DgDx(i));
    if (!outArgs.supports(OUT_ARG_DgDx_dot, i).none())
	me_outargs.set_DgDx(i, outArgs.get_DgDx_dot(i));
    for (int j=0; j<num_p; j++)
      if (!outArgs.supports(OUT_ARG_DgDp, i, j).none())
	me_outargs.set_DgDp(i, j, outArgs.get_DgDp(i,j));
  }

  bool do_quad = false;
  InArgs::sg_const_vector_t x_sg;
  InArgs::sg_const_vector_t x_dot_sg;
  Teuchos::Array<InArgs::sg_const_vector_t> p_sg(num_p);
  OutArgs::sg_vector_t f_sg;
  OutArgs::sg_operator_t W_sg;
  Teuchos::Array<SGDerivative> dfdp_sg(num_p);
  Teuchos::Array<OutArgs::sg_vector_t> g_sg(num_g);
  Teuchos::Array<SGDerivative> dgdx_sg(num_g);
  Teuchos::Array<SGDerivative> dgdx_dot_sg(num_g);
  Teuchos::Array< Teuchos::Array<SGDerivative> > dgdp_sg(num_g);
  TEUCHOS_TEST_FOR_EXCEPTION(inArgs.get_sg_basis() == Teuchos::null, 
		     std::logic_error,
		     "Error!  Stokhos::SGQuadModelEvaluator::evalModel():  " <<
		     "SG basis inArg cannot be null!");
  TEUCHOS_TEST_FOR_EXCEPTION(inArgs.get_sg_quadrature() == Teuchos::null, 
		     std::logic_error,
		     "Error!  Stokhos::SGQuadModelEvaluator::evalModel():  " <<
		     "SG quadrature inArg cannot be null!");
  Teuchos::RCP<const Stokhos::OrthogPolyBasis<int,double> > basis = 
    inArgs.get_sg_basis();
  Teuchos::RCP< const Stokhos::Quadrature<int,double> > quad = 
    inArgs.get_sg_quadrature();
  if (inArgs.supports(IN_ARG_x_sg)) {
    x_sg = inArgs.get_x_sg();
    if (x_sg != Teuchos::null) {
      do_quad = true;
    }
  }
  if (inArgs.supports(IN_ARG_x_dot_sg)) {
    x_dot_sg = inArgs.get_x_dot_sg();
    if (x_dot_sg != Teuchos::null) {
      do_quad = true;
    }
  }
  for (int i=0; i<num_p; i++) {
    p_sg[i] = inArgs.get_p_sg(i);
    if (p_sg[i] != Teuchos::null) {
      do_quad = true;
    }
  }
  if (outArgs.supports(OUT_ARG_f_sg)) {
    f_sg = outArgs.get_f_sg();
    if (f_sg != Teuchos::null)
      f_sg->init(0.0);
  }
  if (outArgs.supports(OUT_ARG_W_sg)) {
    W_sg = outArgs.get_W_sg();
    if (W_sg != Teuchos::null)
      W_sg->init(0.0);
  }
  for (int i=0; i<num_p; i++) {
    if (!outArgs.supports(OUT_ARG_DfDp_sg, i).none()) {
      dfdp_sg[i] = outArgs.get_DfDp_sg(i);
      if (dfdp_sg[i].getMultiVector() != Teuchos::null)
	dfdp_sg[i].getMultiVector()->init(0.0);
      else if (dfdp_sg[i].getLinearOp() != Teuchos::null)
	dfdp_sg[i].getLinearOp()->init(0.0);
    }
  }
      
  for (int i=0; i<num_g; i++) {
    g_sg[i] = outArgs.get_g_sg(i);
    if (g_sg[i] != Teuchos::null)
      g_sg[i]->init(0.0);
    
    if (!outArgs.supports(OUT_ARG_DgDx_sg, i).none()) {
      dgdx_sg[i] = outArgs.get_DgDx_sg(i);
      if (dgdx_sg[i].getMultiVector() != Teuchos::null)
	dgdx_sg[i].getMultiVector()->init(0.0);
      else if (dgdx_sg[i].getLinearOp() != Teuchos::null)
	dgdx_sg[i].getLinearOp()->init(0.0);
    }

    if (!outArgs.supports(OUT_ARG_DgDx_dot_sg, i).none()) {
      dgdx_dot_sg[i] = outArgs.get_DgDx_dot_sg(i);
      if (dgdx_dot_sg[i].getMultiVector() != Teuchos::null)
	dgdx_dot_sg[i].getMultiVector()->init(0.0);
      else if (dgdx_dot_sg[i].getLinearOp() != Teuchos::null)
	dgdx_dot_sg[i].getLinearOp()->init(0.0);
    }

    dgdp_sg[i].resize(num_p);
    for (int j=0; j<num_p; j++) {
      if (!outArgs.supports(OUT_ARG_DgDp_sg, i, j).none()) {
	dgdp_sg[i][j] = outArgs.get_DgDp_sg(i,j);
	if (dgdp_sg[i][j].getMultiVector() != Teuchos::null)
	  dgdp_sg[i][j].getMultiVector()->init(0.0);
	else if (dgdp_sg[i][j].getLinearOp() != Teuchos::null)
	  dgdp_sg[i][j].getLinearOp()->init(0.0);
      }
    }
  }

  if (do_quad) {
    // Get quadrature data
    const Teuchos::Array< Teuchos::Array<double> >& quad_points = 
      quad->getQuadPoints();
    const Teuchos::Array<double>& quad_weights = 
      quad->getQuadWeights();
    const Teuchos::Array< Teuchos::Array<double> > & quad_values = 
      quad->getBasisAtQuadPoints();
    const Teuchos::Array<double>& basis_norms = basis->norm_squared();

    // Perform integrations
    for (int qp=0; qp<quad_points.size(); qp++) {

      // StieltjesPCEBasis can introduce quadrature points with zero weight
      // Don't do those evaluations, since the model might not like the
      // quadrature points (i.e., zero)
      if (quad_weights[qp] == 0.0)
	continue;

      {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
        TEUCHOS_FUNC_TIME_MONITOR_DIFF("Stokhos: SGQuadModelEvaluator -- Polynomial Evaluation",
          PolyEvaluation);
#endif

        // Evaluate inputs at quadrature points
        if (x_sg != Teuchos::null) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
          TEUCHOS_FUNC_TIME_MONITOR("Stokhos: SGQuadModelEvaluator -- X Evaluation");
#endif
          x_sg->evaluate(quad_values[qp], *x_qp);
          me_inargs.set_x(x_qp);
        }
        if (x_dot_sg != Teuchos::null) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
          TEUCHOS_FUNC_TIME_MONITOR("Stokhos: SGQuadModelEvaluator -- X_dot Evaluation");
#endif
          x_dot_sg->evaluate(quad_values[qp], *x_dot_qp);
          me_inargs.set_x_dot(x_qp);
        }
        for (int i=0; i<num_p; i++) {
          if (p_sg[i] != Teuchos::null) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
            TEUCHOS_FUNC_TIME_MONITOR("Stokhos: SGQuadModelEvaluator -- P Evaluation");
#endif
            p_sg[i]->evaluate(quad_values[qp], *(p_qp[i]));
            me_inargs.set_p(i, p_qp[i]);
          }
        }
        if (f_sg != Teuchos::null)
          me_outargs.set_f(f_qp);
        if (W_sg != Teuchos::null)
          me_outargs.set_W(W_qp);
	for (int i=0; i<num_p; i++) {
	  if (!dfdp_sg[i].isEmpty())
	    me_outargs.set_DfDp(i, dfdp_qp[i]);
	}
        for (int i=0; i<num_g; i++) {
	  if (g_sg[i] != Teuchos::null)
	    me_outargs.set_g(i, g_qp[i]);
	  if (!dgdx_dot_sg[i].isEmpty())
	    me_outargs.set_DgDx_dot(i, dgdx_dot_qp[i]);
	  if (!dgdx_sg[i].isEmpty())
	    me_outargs.set_DgDx(i, dgdx_qp[i]);
          for (int j=0; j<num_p; j++)
            if (!dgdp_sg[i][j].isEmpty())
              me_outargs.set_DgDp(i, j, dgdp_qp[i][j]);
        }

      }

      {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
        TEUCHOS_FUNC_TIME_MONITOR("Stokhos: SGQuadModelEvaluator -- Model Evaluation");
#endif

        // Evaluate model at quadrature points
        me->evalModel(me_inargs, me_outargs);

      }

      {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
        TEUCHOS_FUNC_TIME_MONITOR_DIFF(
	  "Stokhos: SGQuadModelEvaluator -- Polynomial Integration", Integration);
#endif

        // Sum in results
        if (f_sg != Teuchos::null) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
          TEUCHOS_FUNC_TIME_MONITOR("Stokhos: SGQuadModelEvaluator -- F Integration");
#endif
          f_sg->sumIntoAllTerms(quad_weights[qp], quad_values[qp], basis_norms,
				*f_qp);
        }
        if (W_sg != Teuchos::null) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
          TEUCHOS_FUNC_TIME_MONITOR("Stokhos: SGQuadModelEvaluator -- W Integration");
#endif
          W_sg->sumIntoAllTerms(quad_weights[qp], quad_values[qp], basis_norms,
				*W_qp);
        }
	for (int j=0; j<num_p; j++) {
	  if (!dfdp_sg[j].isEmpty()) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
	    TEUCHOS_FUNC_TIME_MONITOR(
	      "Stokhos: SGQuadModelEvaluator -- df/dp Integration");
#endif
	    if (dfdp_sg[j].getMultiVector() != Teuchos::null) {
	      dfdp_sg[j].getMultiVector()->sumIntoAllTerms(
		quad_weights[qp], quad_values[qp], basis_norms, 
		*(dfdp_qp[j].getMultiVector()));
	    }
	    else if (dfdp_sg[j].getLinearOp() != Teuchos::null) {
	      dfdp_sg[j].getLinearOp()->sumIntoAllTerms(
		quad_weights[qp], quad_values[qp], basis_norms, 
		*(dfdp_qp[j].getLinearOp()));
	    }
	  }
	}
        for (int i=0; i<num_g; i++) {
          if (g_sg[i] != Teuchos::null) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
            TEUCHOS_FUNC_TIME_MONITOR("Stokhos: SGQuadModelEvaluator -- G Integration");
#endif
            g_sg[i]->sumIntoAllTerms(quad_weights[qp], quad_values[qp], 
				     basis_norms, *g_qp[i]);
          }
	  if (!dgdx_dot_sg[i].isEmpty()) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
	    TEUCHOS_FUNC_TIME_MONITOR(
	      "Stokhos: SGQuadModelEvaluator -- dg/dx_dot Integration");
#endif
	    if (dgdx_dot_sg[i].getMultiVector() != Teuchos::null) {
	      dgdx_dot_sg[i].getMultiVector()->sumIntoAllTerms(
		quad_weights[qp], quad_values[qp], basis_norms, 
		*(dgdx_dot_qp[i].getMultiVector()));
	    }
	    else if (dgdx_dot_sg[i].getLinearOp() != Teuchos::null) {
	      dgdx_dot_sg[i].getLinearOp()->sumIntoAllTerms(
		quad_weights[qp], quad_values[qp], basis_norms, 
		*(dgdx_dot_qp[i].getLinearOp()));
	    }
	  }
	  if (!dgdx_sg[i].isEmpty()) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
	    TEUCHOS_FUNC_TIME_MONITOR(
	      "Stokhos: SGQuadModelEvaluator -- dg/dx Integration");
#endif
	    if (dgdx_sg[i].getMultiVector() != Teuchos::null) {
	      dgdx_sg[i].getMultiVector()->sumIntoAllTerms(
		quad_weights[qp], quad_values[qp], basis_norms, 
		*(dgdx_qp[i].getMultiVector()));
	    }
	    else if (dgdx_sg[i].getLinearOp() != Teuchos::null) {
	      dgdx_sg[i].getLinearOp()->sumIntoAllTerms(
		quad_weights[qp], quad_values[qp], basis_norms, 
		*(dgdx_qp[i].getLinearOp()));
	    }
	  }
          for (int j=0; j<num_p; j++) {
	    if (!dgdp_sg[i][j].isEmpty()) {
#ifdef STOKHOS_TEUCHOS_TIME_MONITOR
	      TEUCHOS_FUNC_TIME_MONITOR(
		"Stokhos: SGQuadModelEvaluator -- dg/dp Integration");
#endif
	      if (dgdp_sg[i][j].getMultiVector() != Teuchos::null) {
		dgdp_sg[i][j].getMultiVector()->sumIntoAllTerms(
		  quad_weights[qp], quad_values[qp], basis_norms, 
		  *(dgdp_qp[i][j].getMultiVector()));
	      }
	      else if (dgdp_sg[i][j].getLinearOp() != Teuchos::null) {
		dgdp_sg[i][j].getLinearOp()->sumIntoAllTerms(
		  quad_weights[qp], quad_values[qp], basis_norms, 
		  *(dgdp_qp[i][j].getLinearOp()));
	      }
	    }
          }
        }

      }
    }
  }
  else {
    // Compute the non-SG functions
    me->evalModel(me_inargs, me_outargs);
  }
}
Beispiel #9
0
void Piro::Epetra::TrapezoidRuleSolver::evalModel( const InArgs& inArgs,
                                     const OutArgs& outArgs ) const
{
  using Teuchos::RCP;
  using Teuchos::rcp;

  EpetraExt::ModelEvaluator::InArgs nox_inargs = noxSolver->createInArgs();
  EpetraExt::ModelEvaluator::OutArgs nox_outargs = noxSolver->createOutArgs();

  // Parse InArgs

  RCP<const Epetra_Vector> p_in;
  if (num_p > 0) {
    p_in = inArgs.get_p(0);
    nox_inargs.set_p(0, p_in);
  }

  // Parse OutArgs: always 1 extra
  RCP<Epetra_Vector> g_out; 
  if (num_g > 0) {
    g_out = outArgs.get_g(0); 
    nox_outargs.set_g(0, g_out);
  }
  RCP<Epetra_Vector> gx_out = outArgs.get_g(num_g); 
  nox_outargs.set_g(num_g, gx_out);


  RCP<Epetra_Vector> x = rcp(new Epetra_Vector(*model->get_x_init()));
  RCP<Epetra_Vector> v = rcp(new Epetra_Vector(*model->get_x_dot_init()));
  RCP<Epetra_Vector> a = rcp(new Epetra_Vector(*model->get_f_map()));
  RCP<Epetra_Vector> x_pred = rcp(new Epetra_Vector(*model->get_f_map()));
  RCP<Epetra_Vector> a_old = rcp(new Epetra_Vector(*model->get_f_map()));

  TEUCHOS_TEST_FOR_EXCEPTION(v == Teuchos::null || x == Teuchos::null, 
                     Teuchos::Exceptions::InvalidParameter,
                     std::endl << "Error in Piro::Epetra::TrapezoidRuleSolver " <<
                     "Requires initial x and x_dot: " << std::endl);
   double nrm;
   v->Norm2(&nrm); *out << "Initial Velocity = " << nrm << endl;

   double t = t_init;

   //calculate intial acceleration using small time step (1.0e-3*delta_t)
   {
     double pert= 1.0e6 * 4.0 / (delta_t * delta_t);
     *x_pred = *x;
     model->injectData(x_pred, x_pred, pert, t);
     noxSolver->evalModel(nox_inargs, nox_outargs);
     a->Update(pert, *gx_out,  -pert, *x_pred,0.0);
     a->Norm2(&nrm); *out << "Calculated a_init = " << nrm << endl;
   }

   // Start integration loop
   double fdt2 = 4.0 / (delta_t * delta_t);
   double dt2f =  delta_t * delta_t / 4.0;
   double hdt  =  delta_t/ 2.0;

   for (int timeStep = 1; timeStep <= numTimeSteps; timeStep++) {
 
     t += delta_t;
 
     *a_old = *a;
     *x_pred = *x;
     x_pred->Update(delta_t, *v, dt2f, *a, 1.0);
     model->injectData(x, x_pred, fdt2, t);

     noxSolver->evalModel(nox_inargs, nox_outargs);
     // Copy out final solution from nonlinear solver
     *x =  *gx_out;
     // Compute a and v and new conditions
     a->Update(fdt2, *x,  -fdt2, *x_pred,0.0);
     v->Update(hdt, *a, hdt, *a_old, 1.0); 

     if (observer != Teuchos::null) observer->observeSolution(*x,t);
     if (g_out != Teuchos::null) 
       g_out->Print(*out << "Responses at time step(time) = " << timeStep << "("<<t<<")\n");
   }
}
void 
QCAD::GenEigensolver::evalModel(const InArgs& inArgs,
			const OutArgs& outArgs ) const
{  
  // type definitions
  typedef Epetra_MultiVector MV;
  typedef Epetra_Operator OP;
  typedef Anasazi::MultiVecTraits<double, Epetra_MultiVector> MVT;
  
  // Get the stiffness and mass matrices
  InArgs model_inArgs = model->createInArgs();
  OutArgs model_outArgs = model->createOutArgs();

  //input args
  model_inArgs.set_t(0.0);

  Teuchos::RCP<const Epetra_Vector> x = model->get_x_init();
  Teuchos::RCP<const Epetra_Vector> x_dot = model->get_x_dot_init();
  model_inArgs.set_x(x);
  model_inArgs.set_x_dot(x_dot);

  model_inArgs.set_alpha(0.0);
  model_inArgs.set_beta(1.0);

  for(int i=0; i<model_num_p; i++)
    model_inArgs.set_p(i, inArgs.get_p(i));
  
  //output args
  Teuchos::RCP<Epetra_CrsMatrix> K = 
    Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(model->create_W(), true);
  model_outArgs.set_W(K); 

  model->evalModel(model_inArgs, model_outArgs); //compute K matrix

  // reset alpha and beta to compute the mass matrix
  model_inArgs.set_alpha(1.0);
  model_inArgs.set_beta(0.0);
  Teuchos::RCP<Epetra_CrsMatrix> M = 
    Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(model->create_W(), true);
  model_outArgs.set_W(M); 

  model->evalModel(model_inArgs, model_outArgs); //compute M matrix

  Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(K->OperatorDomainMap(), blockSize) );
  ivec->Random();

  // Create the eigenproblem.
  Teuchos::RCP<Anasazi::BasicEigenproblem<double, MV, OP> > eigenProblem =
    Teuchos::rcp( new Anasazi::BasicEigenproblem<double, MV, OP>(K, M, ivec) );

  // Inform the eigenproblem that the operator A is symmetric
  eigenProblem->setHermitian(bHermitian);

  // Set the number of eigenvalues requested
  eigenProblem->setNEV( nev );

  // Inform the eigenproblem that you are finishing passing it information
  bool bSuccess = eigenProblem->setProblem();
  TEUCHOS_TEST_FOR_EXCEPTION(!bSuccess, Teuchos::Exceptions::InvalidParameter,
     "Anasazi::BasicEigenproblem::setProblem() returned an error.\n" << std::endl);

  // Create parameter list to pass into the solver manager
  //
  Teuchos::ParameterList eigenPL;
  eigenPL.set( "Which", which );
  eigenPL.set( "Block Size", blockSize );
  eigenPL.set( "Maximum Iterations", maxIters );
  eigenPL.set( "Convergence Tolerance", conv_tol );
  eigenPL.set( "Full Ortho", true );
  eigenPL.set( "Use Locking", true );
  eigenPL.set( "Verbosity", Anasazi::IterationDetails );

  // Create the solver manager
  Anasazi::LOBPCGSolMgr<double, MV, OP> eigenSolverMan(eigenProblem, eigenPL);

  // Solve the problem
  Anasazi::ReturnType returnCode = eigenSolverMan.solve();

  // Get the eigenvalues and eigenvectors from the eigenproblem
  Anasazi::Eigensolution<double,MV> sol = eigenProblem->getSolution();
  std::vector<Anasazi::Value<double> > evals = sol.Evals;
  Teuchos::RCP<MV> evecs = sol.Evecs;

  std::vector<double> evals_real(sol.numVecs);
  for(int i=0; i<sol.numVecs; i++) evals_real[i] = evals[i].realpart;

  // Compute residuals.
  std::vector<double> normR(sol.numVecs);
  if (sol.numVecs > 0) {
    Teuchos::SerialDenseMatrix<int,double> T(sol.numVecs, sol.numVecs);
    Epetra_MultiVector Kvec( K->OperatorDomainMap(), evecs->NumVectors() );
    Epetra_MultiVector Mvec( M->OperatorDomainMap(), evecs->NumVectors() );
    T.putScalar(0.0); 
    for (int i=0; i<sol.numVecs; i++) {
      T(i,i) = evals_real[i];
    }
    K->Apply( *evecs, Kvec );  
    M->Apply( *evecs, Mvec );  
    MVT::MvTimesMatAddMv( -1.0, Mvec, T, 1.0, Kvec );
    MVT::MvNorm( Kvec, normR );
  }

  // Print the results
  std::ostringstream os;
  os.setf(std::ios_base::right, std::ios_base::adjustfield);
  os<<"Solver manager returned " << (returnCode == Anasazi::Converged ? "converged." : "unconverged.") << std::endl;
  os<<std::endl;
  os<<"------------------------------------------------------"<<std::endl;
  os<<std::setw(16)<<"Eigenvalue"
    <<std::setw(18)<<"Direct Residual"
    <<std::endl;
  os<<"------------------------------------------------------"<<std::endl;
  for (int i=0; i<sol.numVecs; i++) {
    os<<std::setw(16)<<evals_real[i]
      <<std::setw(18)<<normR[i]/evals_real[i]
      <<std::endl;
  }
  os<<"------------------------------------------------------"<<std::endl;

  std::cout << Anasazi::Anasazi_Version() << std::endl << std::endl;
  std::cout << os.str();



  // Package the results in an eigendata structure and "observe" them 
  //   (put them into the user-supplied StateManager object)  (see Albany_SaveEigenData.cpp)
  Teuchos::RCP<Albany::EigendataStruct> eigenData = Teuchos::rcp( new Albany::EigendataStruct );
  eigenData->eigenvalueIm = Teuchos::null;  // eigenvalues are real
  eigenData->eigenvectorIm = Teuchos::null; // eigenvectors are real

  Teuchos::RCP<Albany::AbstractDiscretization> disc = 
    observer->getDiscretization();

  eigenData->eigenvalueRe = Teuchos::rcp( new std::vector<double>(evals_real) );
  for(int i=0; i<sol.numVecs; i++) (*(eigenData->eigenvalueRe))[i] *= -1; 
      //make eigenvals --> neg_eigenvals to mimic historic LOCA eigensolver (TODO: remove this and switch convention)

  if (sol.numVecs > 0) {
    // Store *overlapped* eigenvectors in EigendataStruct
    eigenData->eigenvectorRe = 
      Teuchos::rcp(new Epetra_MultiVector(*(disc->getOverlapMap()), sol.numVecs));

    // Importer for overlapped data
    Teuchos::RCP<Epetra_Import> importer =
      Teuchos::rcp(new Epetra_Import(*(disc->getOverlapMap()), *(disc->getMap())));

    // Overlapped eigenstate vectors
    for(int i=0; i<sol.numVecs; i++)
      (*(eigenData->eigenvectorRe))(i)->Import( *((*evecs)(i)), *importer, Insert );
  }

  observer->setEigenData(eigenData);
  
}
Beispiel #11
0
void 
MockModelEval_D::
evalModel(const InArgs& inArgs, const OutArgs& outArgs) const
{
  int proc = comm->MyPID();

  // 
  // Deterministic calculation
  //

  // Parse InArgs
  RCP<const Epetra_Vector> p1_in = inArgs.get_p(0);
  if (p1_in == Teuchos::null)
    p1_in = p1_init;
  RCP<const Epetra_Vector> p2_in = inArgs.get_p(1);
  if (p2_in == Teuchos::null)
    p2_in = p2_init;

  RCP<const Epetra_Vector> x_in = inArgs.get_x();

  // Parse OutArgs
  RCP<Epetra_Vector> f_out = outArgs.get_f(); 
  if (f_out != Teuchos::null) {
    double p = (*p1_in)[0];
    double xi = (*p2_in)[0];
    if (proc == 0) {
      double x = (*x_in)[0];
      (*f_out)[0] = x - p + xi;
    }
  }

  RCP<Epetra_CrsMatrix> W_out = 
    Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(outArgs.get_W()); 
  if (W_out != Teuchos::null) {
    if (proc == 0) {
      double val = 1.0;
      int i = 0;
      W_out->ReplaceMyValues(i, 1, &val, &i);
    }
  }

  RCP<Epetra_MultiVector> dfdp1 = outArgs.get_DfDp(0).getMultiVector();
  if (dfdp1 != Teuchos::null) {
    if (proc == 0)
      (*dfdp1)[0][0] = -1.0;
  }
  RCP<Epetra_MultiVector> dfdp2 = outArgs.get_DfDp(1).getMultiVector();
  if (dfdp2 != Teuchos::null) {
    if (proc == 0)
      (*dfdp2)[0][0] = 1.0;
  }

  RCP<Epetra_Vector> g_out = outArgs.get_g(0); 
  if (g_out != Teuchos::null) {
    if (proc == 0) {
      double x = (*x_in)[0];
      (*g_out)[0] = 1.0 / x;
    }
  }
    

  RCP<Epetra_MultiVector> dgdx = outArgs.get_DgDx(0).getMultiVector();
  if (dgdx != Teuchos::null) {
    if (proc == 0) {
      double x = (*x_in)[0];
      (*dgdx)[0][0] = -1.0 / (x*x);
    }
  }

  RCP<Epetra_MultiVector> dgdp1 = outArgs.get_DgDp(0,0).getMultiVector();
  if (dgdp1 != Teuchos::null) {
    if (proc == 0) {
      (*dgdp1)[0][0] = 0.0;
    }
  }
  RCP<Epetra_MultiVector> dgdp2 = outArgs.get_DgDp(0,1).getMultiVector();
  if (dgdp2 != Teuchos::null) {
    if (proc == 0) {
      (*dgdp2)[0][0] = 0.0;
    }
  }

  // 
  // Stochastic calculation
  //

#ifdef Piro_ENABLE_Stokhos
  // Parse InArgs
  RCP<const Stokhos::OrthogPolyBasis<int,double> > basis = 
    inArgs.get_sg_basis();
  RCP<Stokhos::OrthogPolyExpansion<int,double> > expn = 
    inArgs.get_sg_expansion();
  InArgs::sg_const_vector_t x_sg = inArgs.get_x_sg();
  InArgs::sg_const_vector_t p1_sg = inArgs.get_p_sg(0);
  InArgs::sg_const_vector_t p2_sg = inArgs.get_p_sg(1);

  // Parse OutArgs
  OutArgs::sg_vector_t f_sg = outArgs.get_f_sg();
  if (f_sg != Teuchos::null && proc == 0) {
    for (int block=0; block<f_sg->size(); block++) {
      (*f_sg)[block][0] = 
	(*x_sg)[block][0] - (*p1_sg)[block][0] + (*p2_sg)[block][0];
    }
  }

  OutArgs::sg_operator_t W_sg = outArgs.get_W_sg();
  if (W_sg != Teuchos::null) {
    W_sg->init(0.0);
    Teuchos::RCP<Epetra_CrsMatrix> W = 
      Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_sg->getCoeffPtr(0), 
						  true);
    if (proc == 0) {
      int i = 0;
      double val = 1.0;
      W->ReplaceMyValues(i, 1, &val, &i);
    }
  }

  RCP<Stokhos::EpetraMultiVectorOrthogPoly> dfdp1_sg = 
    outArgs.get_DfDp_sg(0).getMultiVector();
  if (dfdp1_sg != Teuchos::null) {
    dfdp1_sg->init(0.0);
    if (proc == 0) {
      (*dfdp1_sg)[0][0][0] = -1.0;
    }
  }
  RCP<Stokhos::EpetraMultiVectorOrthogPoly> dfdp2_sg = 
    outArgs.get_DfDp_sg(1).getMultiVector();
  if (dfdp2_sg != Teuchos::null) {
    dfdp2_sg->init(0.0);
    if (proc == 0) {
      (*dfdp2_sg)[0][0][0] = 1.0;
    }
  }

  Stokhos::OrthogPolyApprox<int,double> x(basis);
  if (x_sg != Teuchos::null && proc == 0) {
    for (int i=0; i<basis->size(); i++) {
      x[i] = (*x_sg)[i][0];
    }   
  }

  OutArgs::sg_vector_t g_sg = outArgs.get_g_sg(0); 
  if (g_sg != Teuchos::null && proc == 0) {
    Stokhos::OrthogPolyApprox<int,double> xinv(basis);
    expn->divide(xinv, 1.0, x);
    for (int block=0; block<g_sg->size(); block++) {
      (*g_sg)[block][0] = xinv[block];
    }
  }

  RCP<Stokhos::EpetraMultiVectorOrthogPoly> dgdx_sg = 
    outArgs.get_DgDx_sg(0).getMultiVector();
  if (dgdx_sg != Teuchos::null && proc == 0) {
    Stokhos::OrthogPolyApprox<int,double> x2(basis), x2inv(basis);
    expn->times(x2, x, x);
    expn->divide(x2inv, -1.0, x2);
    for (int block=0; block<dgdx_sg->size(); block++) {
      (*dgdx_sg)[block][0][0] = x2inv[block];
    }
  }

  RCP<Stokhos::EpetraMultiVectorOrthogPoly> dgdp1_sg = 
    outArgs.get_DgDp_sg(0,0).getMultiVector();
  if (dgdp1_sg != Teuchos::null) {
    dgdp1_sg->init(0.0);
  }
  RCP<Stokhos::EpetraMultiVectorOrthogPoly> dgdp2_sg = 
    outArgs.get_DgDp_sg(0,1).getMultiVector();
  if (dgdp2_sg != Teuchos::null) {
    dgdp2_sg->init(0.0);
  }
#endif
} 
void EpetraExt::MultiPointModelEvaluator::evalModel( const InArgs& inArgs,
                                            const OutArgs& outArgs ) const
{

  EpetraExt::ModelEvaluator::InArgs  underlyingInArgs  = underlyingME->createInArgs();
  EpetraExt::ModelEvaluator::OutArgs underlyingOutArgs = underlyingME->createOutArgs();

  //temp code for multipoint param q vec
/*
  Teuchos::RefCountPtr<Epetra_Vector> q =
    Teuchos::rcp(new Epetra_Vector(*(underlyingME->get_p_map(1))));
*/

  // Parse InArgs
  Teuchos::RefCountPtr<const Epetra_Vector> p_in = inArgs.get_p(0);
  if (p_in.get()) underlyingInArgs.set_p(0, p_in);

  Teuchos::RefCountPtr<const Epetra_Vector> x_in = inArgs.get_x();
  block_x->Epetra_Vector::operator=(*x_in); //copy into block vector

  // Parse OutArgs
  Teuchos::RefCountPtr<Epetra_Vector> f_out = outArgs.get_f();

  Teuchos::RefCountPtr<Epetra_Operator> W_out = outArgs.get_W();
  Teuchos::RefCountPtr<EpetraExt::BlockCrsMatrix> W_block =
     Teuchos::rcp_dynamic_cast<EpetraExt::BlockCrsMatrix>(W_out);

  Teuchos::RefCountPtr<Epetra_Vector> g_out;
  if (underlyingNg) g_out = outArgs.get_g(0);
  if (g_out.get()) g_out->PutScalar(0.0);

  EpetraExt::ModelEvaluator::Derivative DfDp_out = outArgs.get_DfDp(0);

  EpetraExt::ModelEvaluator::Derivative DgDx_out;
  EpetraExt::ModelEvaluator::Derivative DgDp_out;
  if (underlyingNg) {
    DgDx_out = outArgs.get_DgDx(0);
    DgDp_out = outArgs.get_DgDp(0,0);
    if (!DgDx_out.isEmpty()) DgDx_out.getMultiVector()->PutScalar(0.0);
    if (!DgDp_out.isEmpty()) DgDp_out.getMultiVector()->PutScalar(0.0);
  }

  // For mathcingProblems, g is needed to calc DgDx DgDp, so ask for
  //  g even if it isn't requested.
  bool need_g = g_out.get();
  if (matchingProblem)
    if ( !DgDx_out.isEmpty() || !DgDp_out.isEmpty() ) need_g = true;


  // Begin loop over Points (steps) owned on this proc
  for (int i=0; i < timeStepsOnTimeDomain; i++) {

    // Set MultiPoint parameter vector
    underlyingInArgs.set_p(1, (*q_vec)[i]);

    // Set InArgs
    if(longlong) {
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
      block_x->ExtractBlockValues(*split_x, (*rowIndex_LL)[i]);
#endif
    }
    else {
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
      block_x->ExtractBlockValues(*split_x, (*rowIndex_int)[i]);
#endif
    }
    underlyingInArgs.set_x(split_x);

    // Set OutArgs
    if (f_out.get()) underlyingOutArgs.set_f(split_f);

    if (need_g) underlyingOutArgs.set_g(0, split_g);

    if (W_out.get()) underlyingOutArgs.set_W(split_W);

    if (!DfDp_out.isEmpty()) underlyingOutArgs.set_DfDp(0, *deriv_DfDp);

    if (!DgDx_out.isEmpty()) underlyingOutArgs.set_DgDx(0, *deriv_DgDx);

    if (!DgDp_out.isEmpty()) underlyingOutArgs.set_DgDp(0, 0, *deriv_DgDp);

    //********Eval Model ********/
    underlyingME->evalModel(underlyingInArgs, underlyingOutArgs);
    //********Eval Model ********/

    // If matchingProblem, modify all g-related quantitites G = 0.5*(g-g*)^2 / g*^2
    if (matchingProblem) {
      if (need_g) {
        double diff = (*split_g)[0] -  (*(*matching_vec)[i])[0];
        double nrmlz = fabs((*(*matching_vec)[i])[0]) + 1.0e-6;
        (*split_g)[0] = 0.5 * diff * diff/(nrmlz*nrmlz);
        if (!DgDx_out.isEmpty()) split_DgDx->Scale(diff/(nrmlz*nrmlz));
        if (!DgDp_out.isEmpty()) split_DgDp->Scale(diff/(nrmlz*nrmlz));
      }
    }

    // Repackage block components into global block matrx/vector/multivector
    if(longlong) {
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
      if (f_out.get()) block_f->LoadBlockValues(*split_f, (*rowIndex_LL)[i]);
      if (W_out.get()) W_block->LoadBlock(*split_W, i, 0);
        // note: split_DfDp points inside deriv_DfDp
      if (!DfDp_out.isEmpty()) block_DfDp->LoadBlockValues(*split_DfDp, (*rowIndex_LL)[i]);
      if (!DgDx_out.isEmpty()) block_DgDx->LoadBlockValues(*split_DgDx, (*rowIndex_LL)[i]);
#endif
    }
    else {
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
      if (f_out.get()) block_f->LoadBlockValues(*split_f, (*rowIndex_int)[i]);
      if (W_out.get()) W_block->LoadBlock(*split_W, i, 0);
        // note: split_DfDp points inside deriv_DfDp
      if (!DfDp_out.isEmpty()) block_DfDp->LoadBlockValues(*split_DfDp, (*rowIndex_int)[i]);
      if (!DgDx_out.isEmpty()) block_DgDx->LoadBlockValues(*split_DgDx, (*rowIndex_int)[i]);
#endif
    }

    // Assemble multiple steps on this domain into g and dgdp(0) vectors
    if (g_out.get()) g_out->Update(1.0, *split_g, 1.0);

    if (!DgDp_out.isEmpty())
      DgDp_out.getMultiVector()->Update(1.0, *split_DgDp, 1.0);

  } // End loop over multiPoint steps on this domain/cluster

  //Copy block vectors into *_out vectors of same size
  if (f_out.get()) f_out->operator=(*block_f);
  if (!DfDp_out.isEmpty())
    DfDp_out.getMultiVector()->operator=(*block_DfDp);
  if (!DgDx_out.isEmpty())
    DgDx_out.getMultiVector()->operator=(*block_DgDx);

  //Sum together obj fn contributions from differnt Domains (clusters).
  if (numTimeDomains > 1) {
    double factorToZeroOutCopies = 0.0;
    if (globalComm->SubDomainComm().MyPID()==0) factorToZeroOutCopies = 1.0;
    if (g_out.get()) {
      (*g_out).Scale(factorToZeroOutCopies);
      double* vPtr = &((*g_out)[0]);
      Epetra_Vector tmp = *(g_out.get());
      globalComm->SumAll( &(tmp[0]), vPtr, num_g0);
    }
    if (!DgDp_out.isEmpty()) {
      DgDp_out.getMultiVector()->Scale(factorToZeroOutCopies);
      double* mvPtr = (*DgDp_out.getMultiVector())[0];
      Epetra_MultiVector tmp = *(DgDp_out.getMultiVector());
      globalComm->SumAll(tmp[0], mvPtr, num_dg0dp0);
    }
  }
}
void Piro::Epetra::NOXSolver::evalModel(const InArgs& inArgs,
				const OutArgs& outArgs ) const
{
  // Parse input parameters
  for (int i=0; i<num_p; i++) {
    Teuchos::RCP<const Epetra_Vector> p_in = inArgs.get_p(i);
    if (p_in != Teuchos::null)
      interface->inargs_set_p(p_in, i); // Pass "p_in" through to inargs 
  }

  // Reset initial guess, if the user requests
  if(piroParams->sublist("NOX").get("Reset Initial Guess",false)==true)
    *currentSolution=*model->get_x_init();

  // Solve
  solver->reset(*currentSolution);
  NOX::StatusTest::StatusType status = solver->solve();

  // Print status
  if (status == NOX::StatusTest::Converged) 
    //utils.out() << "Step Converged" << std::endl;
    ;
  else {
    utils.out() << "Nonlinear solver failed to converge!" << std::endl;
    outArgs.setFailed();
  }

  // Get the NOX and Epetra_Vector with the final solution from the solver
  (*currentSolution)=grp->getX();
  Teuchos::RCP<const Epetra_Vector> finalSolution = 
    Teuchos::rcp(&(currentSolution->getEpetraVector()), false);

  // Print solution
  if (utils.isPrintType(NOX::Utils::Details)) {
    utils.out() << std::endl << "Final Solution" << std::endl
		<< "****************" << std::endl;
    finalSolution->Print(utils.pout());
  }

  // Output the parameter list
  if (utils.isPrintType(NOX::Utils::Parameters)) {
    utils.out() << std::endl << "Final Parameters" << std::endl
		<< "****************" << std::endl;
    piroParams->print(utils.out());
    utils.out() << std::endl;
  }

  // Print stats
  bool print_stats = piroParams->get("Print Convergence Stats", true);
  if (print_stats) {
    static int totalNewtonIters=0;
    static int totalKrylovIters=0;
    static int stepNum=0;
    int NewtonIters = piroParams->sublist("NOX").
      sublist("Output").get("Nonlinear Iterations", -1000);

    int KrylovIters = linsys->getLinearItersTotal() - totalKrylovIters;
    int lastSolveKrylovIters = linsys->getLinearItersLastSolve();

    totalNewtonIters += NewtonIters;
    totalKrylovIters += KrylovIters;
    stepNum++;

    utils.out() << "Convergence Stats: for step  #" << stepNum << " : Newton, Krylov, Kr/Ne; LastKrylov, LastTol: " 
	 << NewtonIters << "  " << KrylovIters << "  " 
	 << (double) KrylovIters / (double) NewtonIters << "  " 
         << lastSolveKrylovIters << " " <<  linsys->getAchievedTol() << std::endl;

    if (stepNum > 1)
     utils.out() << "Convergence Stats: running total: Newton, Krylov, Kr/Ne, Kr/Step: " 
           << totalNewtonIters << "  " << totalKrylovIters << "  " 
           << (double) totalKrylovIters / (double) totalNewtonIters 
           << "  " << (double) totalKrylovIters / (double) stepNum << std::endl;
    
  }
    
  //
  // Do Sensitivity Calc, if requested. See 3 main steps 
  //

  // Set inargs and outargs
  EpetraExt::ModelEvaluator::InArgs model_inargs = model->createInArgs();
  EpetraExt::ModelEvaluator::OutArgs model_outargs = model->createOutArgs();
  model_inargs.set_x(finalSolution);

  // We make different choices for layouts of df/dp, dg/dx depending on
  // whether we are doing forward or adjoint sensitivities
  std::string sensitivity_method = piroParams->get("Sensitivity Method",
						   "Forward");

  bool do_sens = false;
  for (int i=0; i<num_p; i++) {
    // p
    model_inargs.set_p(i, inArgs.get_p(i));
    
    // df/dp
    do_sens = false;
    for (int j=0; j<num_g; j++) {
      if (!outArgs.supports(OUT_ARG_DgDp, j, i).none() && 
	  !outArgs.get_DgDp(j,i).isEmpty()) {
	do_sens = true;
        
        // This code does not work with non-empty p_indexes.  The reason is
        // each p_indexes could theoretically be different for each g.
        // We would then need to make one df/dp for all the chosen p's
        // and then index into them properly below.  Note that the number of
        // columns in df/dp should be the number of chosen p's, not the total
        // number of p's.
	if (outArgs.get_DgDp(j,i).getMultiVector() != Teuchos::null) {
	  Teuchos::Array<int> p_indexes = 
	    outArgs.get_DgDp(j,i).getDerivativeMultiVector().getParamIndexes();
	  TEUCHOS_TEST_FOR_EXCEPTION(p_indexes.size() > 0, 
			     Teuchos::Exceptions::InvalidParameter,
			     std::endl <<
			     "Piro::Epetra::NOXSolver::evalModel():  " <<
			     "Non-empty paramIndexes for dg/dp(" << i << "," <<
			     j << ") is not currently supported." << std::endl);
	}
      }
    }
    if (do_sens) {
      Teuchos::RCP<const Epetra_Map> p_map = model->get_p_map(i);
      Teuchos::RCP<const Epetra_Map> f_map = model->get_f_map();
      int num_params = p_map->NumGlobalElements();
      int num_resids = f_map->NumGlobalElements();
      bool p_dist = p_map->DistributedGlobal();
      bool f_dist = f_map->DistributedGlobal();
      DerivativeSupport ds =  model_outargs.supports(OUT_ARG_DfDp,i);
      // Determine which layout to use for df/dp.  Ideally one would look
      // at num_params, num_resids, what is supported by the underlying
      // model evaluator, and the sensitivity method, and make the best 
      // choice to minimze the number of solves.  However this choice depends 
      // also on what layout of dg/dx is supported (e.g., if only the operator 
      // form is supported for forward sensitivities, then df/dp must be
      // DERIV_MV_BY_COL).  For simplicity, we order the conditional tests
      // to get the right layout in most situations.
      DerivativeLayout dfdp_layout;
      if (sensitivity_method == "Forward") {
        if (ds.supports(DERIV_MV_BY_COL) && !p_dist)
          dfdp_layout = COL;
        else if (ds.supports(DERIV_TRANS_MV_BY_ROW) && !f_dist)
          dfdp_layout = ROW;
	else if (ds.supports(DERIV_LINEAR_OP))
          dfdp_layout = OP;
        else
	  TEUCHOS_TEST_FOR_EXCEPTION(
	    true, std::logic_error, 
	    std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
	    "For df/dp(" << i <<") with forward sensitivities, " <<
	    "underlying ModelEvaluator must support DERIV_LINEAR_OP, " <<
	    "DERIV_MV_BY_COL with p not distributed, or "
	    "DERIV_TRANS_MV_BY_ROW with f not distributed." <<
	    std::endl);
      }
      else if (sensitivity_method == "Adjoint") {
        if (ds.supports(DERIV_LINEAR_OP))
          dfdp_layout = OP;
	else if (ds.supports(DERIV_TRANS_MV_BY_ROW) && !f_dist)
          dfdp_layout = ROW;
	else if (ds.supports(DERIV_MV_BY_COL) && !p_dist)
          dfdp_layout = COL;
        else
	  TEUCHOS_TEST_FOR_EXCEPTION(
	    true, std::logic_error, 
	    std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
	    "For df/dp(" << i <<") with adjoint sensitivities, " <<
	    "underlying ModelEvaluator must support DERIV_LINEAR_OP, " <<
	    "DERIV_MV_BY_COL with p not distributed, or "
	    "DERIV_TRANS_MV_BY_ROW with f not distributed." <<
	    std::endl);
      }
      else
        TEUCHOS_TEST_FOR_EXCEPTION(true, 
       		           Teuchos::Exceptions::InvalidParameter,
		           std::endl <<
		           "Piro::Epetra::NOXSolver::evalModel():  " <<
		           "Unknown sensitivity method" << sensitivity_method <<
		           ".  Valid choices are \"Forward\" and \"Adjoint\"." 
                           << std::endl);

      if (dfdp_layout == COL) {
	Teuchos::RCP<Epetra_MultiVector> dfdp =
	  Teuchos::rcp(new Epetra_MultiVector(*f_map, num_params));
	// Teuchos::Array<int> p_indexes = 
	// 	outArgs.get_DgDp(i,0).getDerivativeMultiVector().getParamIndexes();
	// EpetraExt::ModelEvaluator::DerivativeMultiVector 
	// 	dmv_dfdp(dfdp, DERIV_MV_BY_COL, p_indexes);
	EpetraExt::ModelEvaluator::DerivativeMultiVector 
	  dmv_dfdp(dfdp, DERIV_MV_BY_COL);
	model_outargs.set_DfDp(i,dmv_dfdp);
      }
      else if (dfdp_layout == ROW) {
	Teuchos::RCP<Epetra_MultiVector> dfdp =
	  Teuchos::rcp(new Epetra_MultiVector(*p_map, num_resids));
	// Teuchos::Array<int> p_indexes = 
	// 	outArgs.get_DgDp(i,0).getDerivativeMultiVector().getParamIndexes();
	// EpetraExt::ModelEvaluator::DerivativeMultiVector 
	// 	dmv_dfdp(dfdp, DERIV_TRANS_MV_BY_ROW, p_indexes);
	EpetraExt::ModelEvaluator::DerivativeMultiVector 
	  dmv_dfdp(dfdp, DERIV_TRANS_MV_BY_ROW);
	model_outargs.set_DfDp(i,dmv_dfdp);
      }
      else if (dfdp_layout == OP) {
	Teuchos::RCP<Epetra_Operator> dfdp_op = model->create_DfDp_op(i);
	TEUCHOS_TEST_FOR_EXCEPTION(
	  dfdp_op == Teuchos::null, std::logic_error, 
	  std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
	  "Needed df/dp operator (" << i << ") is null!" << std::endl);
	model_outargs.set_DfDp(i,dfdp_op);
      }
    }
  }

  for (int j=0; j<num_g; j++) {
    // g
    Teuchos::RCP<Epetra_Vector> g_out = outArgs.get_g(j);
    if (g_out != Teuchos::null) {
      g_out->PutScalar(0.0);
      model_outargs.set_g(j, g_out);
    }

    // dg/dx
    do_sens = false;
    for (int i=0; i<num_p; i++) {
      Teuchos::RCP<Epetra_MultiVector> dgdp_out;
      if (!outArgs.supports(OUT_ARG_DgDp, j, i).none() &&
	  !outArgs.get_DgDp(j,i).isEmpty()) {
	do_sens = true;
      }
    }
    if (do_sens) {
      Teuchos::RCP<const Epetra_Map> g_map = model->get_g_map(j);
      Teuchos::RCP<const Epetra_Map> x_map = model->get_x_map();
      int num_responses = g_map->NumGlobalElements();
      int num_solution = x_map->NumGlobalElements();
      bool g_dist = g_map->DistributedGlobal();
      bool x_dist = x_map->DistributedGlobal();
      DerivativeSupport ds =  model_outargs.supports(OUT_ARG_DgDx,j);
      DerivativeLayout dgdx_layout;
      if (sensitivity_method == "Forward") {
	if (ds.supports(DERIV_LINEAR_OP))
          dgdx_layout = OP;
        else if (ds.supports(DERIV_MV_BY_COL) && !x_dist)
          dgdx_layout = COL;
        else if (ds.supports(DERIV_TRANS_MV_BY_ROW) && !g_dist)
          dgdx_layout = ROW;
	else
	  TEUCHOS_TEST_FOR_EXCEPTION(
	    true, std::logic_error, 
	    std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
	    "For dg/dx(" << j <<") with forward sensitivities, " <<
	    "underlying ModelEvaluator must support DERIV_LINEAR_OP, " <<
	    "DERIV_MV_BY_COL with x not distributed, or "
	    "DERIV_TRANS_MV_BY_ROW with g not distributed." <<
	    std::endl);
      }
      else if (sensitivity_method == "Adjoint") {
	if (ds.supports(DERIV_TRANS_MV_BY_ROW) && !g_dist)
          dgdx_layout = ROW;
	else if (ds.supports(DERIV_MV_BY_COL) && !x_dist)
          dgdx_layout = COL;
	else if (ds.supports(DERIV_LINEAR_OP))
          dgdx_layout = OP;
        else
	  TEUCHOS_TEST_FOR_EXCEPTION(
	    true, std::logic_error, 
	    std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
	    "For dg/dx(" << j <<") with adjoint sensitivities, " <<
	    "underlying ModelEvaluator must support DERIV_LINEAR_OP, " <<
	    "DERIV_MV_BY_COL with x not distributed, or "
	    "DERIV_TRANS_MV_BY_ROW with g not distributed." <<
	    std::endl);
      }
      else
        TEUCHOS_TEST_FOR_EXCEPTION(true, 
       		           Teuchos::Exceptions::InvalidParameter,
		           std::endl <<
		           "Piro::Epetra::NOXSolver::evalModel():  " <<
		           "Unknown sensitivity method" << sensitivity_method <<
		           ".  Valid choices are \"Forward\" and \"Adjoint\"." 
                           << std::endl);

      if (dgdx_layout == OP) {
	Teuchos::RCP<Epetra_Operator> dgdx_op = model->create_DgDx_op(j);
	TEUCHOS_TEST_FOR_EXCEPTION(
	  dgdx_op == Teuchos::null, std::logic_error, 
	  std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
	  "Needed dg/dx operator (" << j << ") is null!" << std::endl);
	model_outargs.set_DgDx(j,dgdx_op);
      }
      else if (dgdx_layout == ROW) {
	Teuchos::RCP<Epetra_MultiVector> dgdx = 
	  Teuchos::rcp(new Epetra_MultiVector(*x_map, num_responses));
	EpetraExt::ModelEvaluator::DerivativeMultiVector 
	  dmv_dgdx(dgdx, DERIV_TRANS_MV_BY_ROW);
	model_outargs.set_DgDx(j,dmv_dgdx);
      }
      else if (dgdx_layout == COL) {
	Teuchos::RCP<Epetra_MultiVector> dgdx = 
	  Teuchos::rcp(new Epetra_MultiVector(*g_map, num_solution));
	EpetraExt::ModelEvaluator::DerivativeMultiVector 
	  dmv_dgdx(dgdx, DERIV_MV_BY_COL);
	model_outargs.set_DgDx(j,dmv_dgdx);
      }

      // dg/dp
      for (int i=0; i<num_p; i++) {
	if (!outArgs.supports(OUT_ARG_DgDp,j,i).none()) {
	  Derivative dgdp = outArgs.get_DgDp(j,i);
	  if (dgdp.getLinearOp() != Teuchos::null) {
	    Teuchos::RCP<const Epetra_Map> g_map = model->get_g_map(j);
	    Teuchos::RCP<const Epetra_Map> p_map = model->get_p_map(i);
	    int num_responses = g_map->NumGlobalElements();
	    int num_params = p_map->NumGlobalElements();
	    bool g_dist = g_map->DistributedGlobal();
	    bool p_dist = p_map->DistributedGlobal();
	    DerivativeSupport ds = model_outargs.supports(OUT_ARG_DgDp,j,i);
	    if (ds.supports(DERIV_LINEAR_OP)) {
	      Teuchos::RCP<Epetra_Operator> dgdp_op = 
		model->create_DgDp_op(j,i);
	      TEUCHOS_TEST_FOR_EXCEPTION(
		dgdp_op == Teuchos::null, std::logic_error, 
		std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
		"Needed dg/dp operator (" << j << "," << i << ") is null!" << 
		std::endl);
	      model_outargs.set_DgDp(j,i,dgdp_op);
	    }
	    else if (ds.supports(DERIV_MV_BY_COL) && !p_dist) {
	      Teuchos::RCP<Epetra_MultiVector> dgdp =
		Teuchos::rcp(new Epetra_MultiVector(*g_map, num_params));
	      EpetraExt::ModelEvaluator::DerivativeMultiVector 
		dmv_dgdp(dgdp, DERIV_MV_BY_COL);
	      model_outargs.set_DgDp(j,i,dmv_dgdp);
	    }
	    else if (ds.supports(DERIV_TRANS_MV_BY_ROW) && !g_dist) {
	      Teuchos::RCP<Epetra_MultiVector> dgdp =
		Teuchos::rcp(new Epetra_MultiVector(*p_map, num_responses));
	      EpetraExt::ModelEvaluator::DerivativeMultiVector 
		dmv_dgdp(dgdp, DERIV_TRANS_MV_BY_ROW);
	      model_outargs.set_DgDp(j,i,dmv_dgdp);
	    }
	    else
	      TEUCHOS_TEST_FOR_EXCEPTION(
		true, std::logic_error, 
		std::endl << "Piro::Epetra::NOXSolver::evalModel():  " << 
		"For dg/dp(" << j << "," << i <<
		") with operator sensitivities, "<<
		"underlying ModelEvaluator must support DERIV_LINEAR_OP, " <<
		"DERIV_MV_BY_COL with p not distributed, or "
		"DERIV_TRANS_MV_BY_ROW with g not distributed." <<
		std::endl);
	  }
	  else
	    model_outargs.set_DgDp(j,i,outArgs.get_DgDp(j,i));
	}
      }
    }
  }
    
  // (1) Calculate g, df/dp, dg/dp, dg/dx
  model->evalModel(model_inargs, model_outargs);

  // Ensure Jacobian is up-to-date
  if (do_sens)
    grp->computeJacobian();

  // Handle operator dg/dp
  for (int i=0; i<num_p; i++) {
    for (int j=0; j<num_g; j++) {
      if (!outArgs.supports(OUT_ARG_DgDp, j, i).none()) {
	if (outArgs.get_DgDp(j,i).getLinearOp() != Teuchos::null) {
	  Teuchos::RCP<Epetra_Operator> op = 
	    outArgs.get_DgDp(j,i).getLinearOp();
	  Teuchos::RCP<SensitivityOperator> sens_op =
	    Teuchos::rcp_dynamic_cast<SensitivityOperator>(op);
	  sens_op->setup(model_outargs.get_DfDp(i), 
			 model_outargs.get_DgDx(j),
			 model_outargs.get_DgDp(j,i), 
			 piroParams, grp, tls_strategy);
	}
      }
    }
  }
    
  if (sensitivity_method == "Forward") {
    for (int i=0; i<num_p; i++) {

      // See if there are any forward sensitivities we need to do
      // that aren't handled by the operator
      do_sens = false;
      for (int j=0; j<num_g; j++) {
	if (!outArgs.supports(OUT_ARG_DgDp, j, i).none()) {
	  if (outArgs.get_DgDp(j,i).getMultiVector() != Teuchos::null)
	    do_sens = true;
	}
      }
      if (!do_sens)
	continue;

      if (!model_outargs.supports(OUT_ARG_DfDp, i).none()) {
	TEUCHOS_TEST_FOR_EXCEPTION(
	  model_outargs.get_DfDp(i).getLinearOp()!=Teuchos::null,
	  std::logic_error,
	  std::endl <<"Piro::Epetra::NOXSolver::evalModel():  " <<
	  "Can\'t use df/dp operator " << i << " with non-operator " <<
	  "forward sensitivities." << std::endl);
	Teuchos::RCP<Epetra_MultiVector> dfdp  = 
	  model_outargs.get_DfDp(i).getMultiVector();
	if (dfdp != Teuchos::null) {
	  int num_cols = dfdp->NumVectors();
	
	  // (2) Calculate dx/dp multivector from -(J^{-1}*df/dp)
	  Teuchos::RCP<Epetra_MultiVector> dxdp = 
	    Teuchos::rcp(new Epetra_MultiVector(dfdp->Map(), num_cols));
	  NOX::Epetra::MultiVector dfdp_nox(
	    dfdp, NOX::DeepCopy,  
	    NOX::Epetra::MultiVector::CreateView);
	  NOX::Epetra::MultiVector dxdp_nox(
	    dxdp, NOX::DeepCopy,  
	    NOX::Epetra::MultiVector::CreateView);
	  
	  grp->applyJacobianInverseMultiVector(*piroParams, dfdp_nox, dxdp_nox);
	  dxdp_nox.scale(-1.0);
	
	  // (3) Calculate dg/dp = dg/dx*dx/dp + dg/dp
	  for (int j=0; j<num_g; j++) {
	    if (!outArgs.supports(OUT_ARG_DgDp, j, i).none()) {
	      Teuchos::RCP<Epetra_MultiVector> dgdp_out = 
		outArgs.get_DgDp(j,i).getMultiVector();
	      if (dgdp_out != Teuchos::null) {
		Derivative dgdx_dv = model_outargs.get_DgDx(j);
		Teuchos::RCP<Epetra_Operator> dgdx_op = dgdx_dv.getLinearOp();
		Teuchos::RCP<Epetra_MultiVector> dgdx = 
		  dgdx_dv.getMultiVector();
		Derivative dfdp_dv = model_outargs.get_DfDp(i);
		EDerivativeMultiVectorOrientation dgdp_orient =
		  outArgs.get_DgDp(j,i).getMultiVectorOrientation();
		if (dgdx_op != Teuchos::null) {
		  bool transpose = false;
		  if (dgdp_orient == DERIV_TRANS_MV_BY_ROW)
		    transpose = true;
		  Epetra_MultiVector tmp(dgdx_op->OperatorRangeMap(),
					 dxdp->NumVectors());
		  dgdx_op->Apply(*dxdp, tmp);
		  if (transpose) {
		    TEUCHOS_TEST_FOR_EXCEPTION(
		      dgdp_out->Map().DistributedGlobal(), 
		      std::logic_error,
		      std::endl << 
		      "Piro::Epetra::NOXSolver::evalModel():  " <<
		      "Can\'t handle special case:  " << 
		      " dg/dx operator, " <<
		      " transposed, distributed dg/dp. " << std::endl);
		    for (int j=0; j<dgdp_out->NumVectors(); j++)
		      for (int i=0; i<dgdp_out->MyLength(); i++)
			(*dgdp_out)[j][i] += tmp[i][j];
		  }
		  else
		    dgdp_out->Update(1.0, tmp, 1.0);
		}
		else {
		  Teuchos::RCP<Epetra_MultiVector> arg1, arg2;
		  EDerivativeMultiVectorOrientation dfdp_orient =
		    model_outargs.get_DfDp(i).getMultiVectorOrientation();
		  EDerivativeMultiVectorOrientation dgdx_orient =
		    model_outargs.get_DgDx(j).getMultiVectorOrientation();
		  char flag1, flag2;
		  if (dgdp_orient == DERIV_MV_BY_COL) {
		    arg1 = dgdx;
		    arg2 = dxdp;
		    if (dgdx_orient == DERIV_MV_BY_COL)
		      flag1 = 'N';
		    else
		      flag1 = 'T';
		    if (dfdp_orient == DERIV_MV_BY_COL)
		      flag2 = 'N';
		    else
		      flag2 = 'T';
		  }
		  else {
		    arg1 = dxdp;
		    arg2 = dgdx;
		    if (dfdp_orient == DERIV_MV_BY_COL)
		      flag1 = 'T';
		    else
		      flag1 = 'N';
		    if (dgdx_orient == DERIV_MV_BY_COL)
		      flag2 = 'T';
		    else
		      flag2 = 'N';
		  }
		  dgdp_out->Multiply(flag1, flag2, 1.0, *arg1, *arg2, 1.0);
		}
	      }
	    }
	  }
	}
      }
    }
  }

  else if (sensitivity_method == "Adjoint") {
    
    // Hold on to original Jacobian operator
    Teuchos::RCP<NOX::Epetra::LinearSystem> linSys = grp->getLinearSystem();
    Teuchos::RCP<Epetra_Operator> jac = linSys->getJacobianOperator();

    tls_strategy->createJacobianTranspose();
    const NOX::Epetra::Vector& x_nox = 
      dynamic_cast<const NOX::Epetra::Vector&>(grp->getX());
    tls_strategy->createTransposePreconditioner(x_nox, *piroParams);

    for (int j=0; j<num_g; j++) {

      // See if there are any forward sensitivities we need to do
      // that aren't handled by the operator
      do_sens = false;
      for (int i=0; i<num_p; i++) {
	if (!outArgs.supports(OUT_ARG_DgDp, j, i).none()) {
	  if (outArgs.get_DgDp(j,i).getMultiVector() != Teuchos::null)
	    do_sens = true;
	}
      }
      if (!do_sens)
	continue;

      if (!model_outargs.supports(OUT_ARG_DgDx, j).none()) {
	TEUCHOS_TEST_FOR_EXCEPTION(
	  model_outargs.get_DgDx(j).getLinearOp()!=Teuchos::null,
	  std::logic_error,
	  std::endl << "Piro::Epetra::NOXSolver::evalModel():  " <<
	  "Can\'t use dg/dx operator " << j << " with non-operator " << 
	  "adjoint sensitivities." << std::endl);
	Teuchos::RCP<Epetra_MultiVector> dgdx  = 
	  model_outargs.get_DgDx(j).getMultiVector();
	if (dgdx != Teuchos::null) {
	  int num_cols = dgdx->NumVectors();
	
	  // (2) Calculate xbar multivector from -(J^{-T}*dg/dx)
	  Teuchos::RCP<Epetra_MultiVector> xbar = 
	    Teuchos::rcp(new Epetra_MultiVector(dgdx->Map(), num_cols));
	  for (int col=0; col<num_cols; col++) {
	    Teuchos::RCP<Epetra_Vector> gx =
	      Teuchos::rcp((*dgdx)(col),false);
	    Teuchos::RCP<Epetra_Vector> xb =
	      Teuchos::rcp((*xbar)(col),false);
	    NOX::Epetra::Vector dgdx_nox(
	      gx, NOX::Epetra::Vector::CreateView, NOX::DeepCopy);
	    NOX::Epetra::Vector xbar_nox(
	      xb, NOX::Epetra::Vector::CreateView, NOX::DeepCopy);
	  
	    // Solve
	    tls_strategy->applyJacobianTransposeInverse(
	      *piroParams, dgdx_nox, xbar_nox);
	  }
	  xbar->Scale(-1.0);
	
	  // (3) Calculate dg/dp^T = df/dp^T*xbar + dg/dp^T
	  for (int i=0; i<num_p; i++) {
	    if (!outArgs.supports(OUT_ARG_DgDp, j, i).none()) {
	      Teuchos::RCP<Epetra_MultiVector> dgdp_out = 
		outArgs.get_DgDp(j,i).getMultiVector();
	      if (dgdp_out != Teuchos::null) {
		Derivative dfdp_dv = model_outargs.get_DfDp(i);
		Teuchos::RCP<Epetra_Operator> dfdp_op = dfdp_dv.getLinearOp();
		Teuchos::RCP<Epetra_MultiVector> dfdp = 
		  dfdp_dv.getMultiVector();
		Derivative dgdx_dv = model_outargs.get_DgDx(j);
		EDerivativeMultiVectorOrientation dgdp_orient =
		  outArgs.get_DgDp(j,i).getMultiVectorOrientation();
		if (dfdp_op != Teuchos::null) {
		  bool transpose = false;
		  if (dgdp_orient == DERIV_MV_BY_COL)
		    transpose = true;
		  Epetra_MultiVector tmp(dfdp_op->OperatorDomainMap(),
					 xbar->NumVectors());
		  dfdp_op->SetUseTranspose(true);
		  dfdp_op->Apply(*xbar, tmp);
		  dfdp_op->SetUseTranspose(false);
		  if (transpose) {
		    TEUCHOS_TEST_FOR_EXCEPTION(
		      dgdp_out->Map().DistributedGlobal(), 
		      std::logic_error,
		      std::endl <<
		      "Piro::Epetra::NOXSolver::evalModel():  " <<
		      "Can\'t handle special case:  " << 
		      " df/dp operator, " <<
		      " transposed, distributed dg/dp. " << std::endl);
		    for (int j=0; j<dgdp_out->NumVectors(); j++)
		      for (int i=0; i<dgdp_out->MyLength(); i++)
			(*dgdp_out)[j][i] += tmp[i][j];
		  }
		  else
		    dgdp_out->Update(1.0, tmp, 1.0);
		}
		else {
		  Teuchos::RCP<Epetra_MultiVector> arg1, arg2;
		  EDerivativeMultiVectorOrientation dgdp_orient =
		    model_outargs.get_DgDp(j,i).getMultiVectorOrientation();
		  EDerivativeMultiVectorOrientation dfdp_orient =
		    model_outargs.get_DfDp(i).getMultiVectorOrientation();
		  EDerivativeMultiVectorOrientation dgdx_orient =
		    model_outargs.get_DgDx(j).getMultiVectorOrientation();
		  char flag1, flag2;
		  if (dgdp_orient == DERIV_TRANS_MV_BY_ROW) {
		    arg1 = dfdp;
		    arg2 = xbar;
		    if (dfdp_orient == DERIV_TRANS_MV_BY_ROW)
		      flag1 = 'N';
		    else
		      flag1 = 'T';
		    if (dgdx_orient == DERIV_TRANS_MV_BY_ROW)
		      flag2 = 'N';
		    else
		      flag2 = 'T';
		  }
		  else {
		    arg1 = xbar;
		    arg2 = dfdp;
		    if (dgdx_orient == DERIV_TRANS_MV_BY_ROW)
		      flag1 = 'T';
		    else
		      flag1 = 'N';
		    if (dfdp_orient == DERIV_TRANS_MV_BY_ROW)
		      flag2 = 'T';
		    else
		      flag2 = 'N';
		    
		  }
		  dgdp_out->Multiply(flag1, flag2, 1.0, *arg1, *arg2, 1.0);
		}
	      }
	    }
	  }
	}
      }
    }

    // Set original operators in linear system
    jac->SetUseTranspose(false);
    linSys->setJacobianOperatorForSolve(jac);
    linSys->destroyPreconditioner();
  }

  if (status == NOX::StatusTest::Converged) 
    if (observer != Teuchos::null)
      observer->observeSolution(*finalSolution);

  // return the final solution as an additional g-vector, if requested
  Teuchos::RCP<Epetra_Vector> gx_out = outArgs.get_g(num_g); 
  if (gx_out != Teuchos::null)  *gx_out = *finalSolution; 

  // Clear RCPs to parameter vectors
  for (int i=0; i<num_p; i++)
    interface->inargs_set_p(Teuchos::null, i); 
 
}