Example #1
0
// *****************************************************************
// ***************************************************************** 
bool LOCA::Epetra::ModelEvaluatorInterface::
computeJacobian(const Epetra_Vector& x, Epetra_Operator& Jac)
{
  // Create inargs
  EpetraExt::ModelEvaluator::InArgs inargs = model_->createInArgs();
  inargs.set_x(Teuchos::rcp(&x, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_alpha))
    inargs.set_alpha(0.0);
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_beta))
    inargs.set_beta(1.0);
  alpha_prev = 0.0; beta_prev = 1.0; // prec must know alpha and beta
  inargs.set_p(0, Teuchos::rcp(&param_vec, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_x_dot)) {
    // Create x_dot, filled with zeros
    if (x_dot == NULL)
      x_dot = new Epetra_Vector(x.Map());
    inargs.set_x_dot(Teuchos::rcp(x_dot, false));
  }

  // Create outargs
  EpetraExt::ModelEvaluator::OutArgs outargs = model_->createOutArgs();
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> eval_f;
  outargs.set_f(eval_f);
  outargs.set_W(Teuchos::rcp(&Jac, false));

  model_->evalModel(inargs, outargs);

  return true;
}
Example #2
0
// *****************************************************************
// *****************************************************************
bool LOCA::Epetra::ModelEvaluatorInterface::
computeShiftedMatrix(double alpha, double beta, const Epetra_Vector& x,
		     Epetra_Operator& A)
{
  // Create inargs
  EpetraExt::ModelEvaluator::InArgs inargs = model_->createInArgs();
  inargs.set_x(Teuchos::rcp(&x, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_alpha))
    inargs.set_alpha(-beta); // alpha and beta are switched between LOCA and Thyra
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_beta))
    inargs.set_beta(alpha);
  alpha_prev = -beta; beta_prev = alpha; // prec must know alpha and beta
  inargs.set_p(0, Teuchos::rcp(&param_vec, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_x_dot)) {
    // Create x_dot, filled with zeros
    if (x_dot == NULL)
      x_dot = new Epetra_Vector(x.Map());
    inargs.set_x_dot(Teuchos::rcp(x_dot, false));
  }

  // Create outargs
  EpetraExt::ModelEvaluator::OutArgs outargs = model_->createOutArgs();
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> eval_f;
  outargs.set_f(eval_f);
  outargs.set_W(Teuchos::rcp(&A, false));

  model_->evalModel(inargs, outargs);

  return true;
}
Example #3
0
// *****************************************************************
// *****************************************************************
bool LOCA::Epetra::ModelEvaluatorInterface::
computeF(const Epetra_Vector& x, Epetra_Vector& F, const FillType fillFlag)
{
  // Create inargs
  EpetraExt::ModelEvaluator::InArgs inargs = model_->createInArgs();
  inargs.set_x(Teuchos::rcp(&x, false));
  inargs.set_p(0, Teuchos::rcp(&param_vec, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_x_dot)) {
    // Create x_dot, filled with zeros
    if (x_dot == NULL)
      x_dot = new Epetra_Vector(x.Map());
    inargs.set_x_dot(Teuchos::rcp(x_dot, false));
  }
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_alpha))
    inargs.set_alpha(0.0);
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_beta))
    inargs.set_beta(1.0);

  // Create outargs
  EpetraExt::ModelEvaluator::OutArgs outargs = model_->createOutArgs();
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> eval_f;
  Teuchos::RefCountPtr<Epetra_Vector> f = Teuchos::rcp(&F, false);
  if (fillFlag == NOX::Epetra::Interface::Required::Residual)
    eval_f.reset(f, EpetraExt::ModelEvaluator::EVAL_TYPE_EXACT); 
  else if (fillFlag == NOX::Epetra::Interface::Required::Jac)
    eval_f.reset(f, EpetraExt::ModelEvaluator::EVAL_TYPE_APPROX_DERIV);
  else
    eval_f.reset(f, EpetraExt::ModelEvaluator::EVAL_TYPE_VERY_APPROX_DERIV);
  outargs.set_f(eval_f);

  model_->evalModel(inargs, outargs);

  return true;
}
Example #4
0
NOX::Abstract::Group::ReturnType 
LOCA::Epetra::ModelEvaluatorInterface::
computeDfDp(LOCA::MultiContinuation::AbstractGroup& grp, 
	    const vector<int>& param_ids,
	    NOX::Abstract::MultiVector& result,
	    bool isValidF) const
{
  // Break result into f and df/dp
  NOX::Epetra::Vector& f = dynamic_cast<NOX::Epetra::Vector&>(result[0]);
  Epetra_Vector& epetra_f = f.getEpetraVector();

  std::vector<int> dfdp_index(result.numVectors()-1);
  for (unsigned int i=0; i<dfdp_index.size(); i++)
    dfdp_index[i] = i+1;
  Teuchos::RefCountPtr<NOX::Epetra::MultiVector> dfdp =
    Teuchos::rcp_dynamic_cast<NOX::Epetra::MultiVector>(result.subView(dfdp_index));
  Epetra_MultiVector& epetra_dfdp = dfdp->getEpetraMultiVector();

  // Create inargs
  EpetraExt::ModelEvaluator::InArgs inargs = model_->createInArgs();
  const NOX::Epetra::Vector& x = 
    dynamic_cast<const NOX::Epetra::Vector&>(grp.getX());
  const Epetra_Vector& epetra_x = x.getEpetraVector();
  inargs.set_x(Teuchos::rcp(&epetra_x, false));
  inargs.set_p(0, Teuchos::rcp(&param_vec, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_x_dot)) {
    // Create x_dot, filled with zeros
    if (x_dot == NULL)
      x_dot = new Epetra_Vector(epetra_x.Map());
    inargs.set_x_dot(Teuchos::rcp(x_dot, false));
  }

  // Create outargs
  EpetraExt::ModelEvaluator::OutArgs outargs = model_->createOutArgs();
  if (!isValidF) {
    EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> eval_f;
    Teuchos::RefCountPtr<Epetra_Vector> F = Teuchos::rcp(&epetra_f, false);
    eval_f.reset(F, EpetraExt::ModelEvaluator::EVAL_TYPE_EXACT); 
    outargs.set_f(eval_f);
  }
  Teuchos::RefCountPtr<Epetra_MultiVector> DfDp = 
    Teuchos::rcp(&epetra_dfdp, false);
  Teuchos::Array<int> param_indexes(param_ids.size());
  for (unsigned int i=0; i<param_ids.size(); i++)
    param_indexes[i] = param_ids[i];
  EpetraExt::ModelEvaluator::DerivativeMultiVector dmv(DfDp, EpetraExt::ModelEvaluator::DERIV_MV_BY_COL,
						       param_indexes);
  EpetraExt::ModelEvaluator::Derivative deriv(dmv);
  outargs.set_DfDp(0, deriv);

  model_->evalModel(inargs, outargs);

  return NOX::Abstract::Group::Ok;
}
Example #5
0
void
LOCA::Epetra::ModelEvaluatorInterface::
postProcessContinuationStep(
          LOCA::Abstract::Iterator::StepStatus stepStatus,
          LOCA::Epetra::Group& group)
{
  // Evaluate responses in model evaluator after successful step
  if (stepStatus != LOCA::Abstract::Iterator::Successful) return;

  // Create inargs
  const NOX::Epetra::Vector& ex = dynamic_cast<const NOX::Epetra::Vector&>(group.getX());
  const Epetra_Vector& x = ex.getEpetraVector();

  EpetraExt::ModelEvaluator::InArgs inargs = model_->createInArgs();
  inargs.set_x(Teuchos::rcp(&x, false));
  inargs.set_p(0, Teuchos::rcp(&param_vec, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_x_dot)) {
    // Create x_dot, filled with zeros
    if (x_dot == NULL)
      x_dot = new Epetra_Vector(x.Map());
    inargs.set_x_dot(Teuchos::rcp(x_dot, false));
  }
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_alpha))
    inargs.set_alpha(0.0);
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_beta))
    inargs.set_beta(1.0);

  // Create outargs
  EpetraExt::ModelEvaluator::OutArgs outargs = model_->createOutArgs();
  int num_g = outargs.Ng();
  if (num_g > 0) {
    Teuchos::RCP<Epetra_Vector> g0 = Teuchos::rcp(new Epetra_Vector(*(model_->get_g_map(0))));

    outargs.set_g(0,g0);

    model_->evalModel(inargs, outargs);
  }
}
Example #6
0
// *****************************************************************
// *****************************************************************
bool LOCA::Epetra::ModelEvaluatorInterface::
computePreconditioner(const Epetra_Vector& x, 
		      Epetra_Operator& M,
		      Teuchos::ParameterList* precParams)
{
  // Create inargs
  EpetraExt::ModelEvaluator::InArgs inargs = model_->createInArgs();
  inargs.set_x(Teuchos::rcp(&x, false));

  // alpha and beta are stored from previous matrix computation
  // which might have been computeJacobian or computeShiftedMatrix
  // This is a state-full hack, but needed since this function
  // does not take alpha and beta as arguments. [AGS 01/10]
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_alpha))
    inargs.set_alpha(alpha_prev);
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_beta))
    inargs.set_beta(beta_prev);
  inargs.set_p(0, Teuchos::rcp(&param_vec, false));
  if (inargs.supports(EpetraExt::ModelEvaluator::IN_ARG_x_dot)) {
    // Create x_dot, filled with zeros
    if (x_dot == NULL)
      x_dot = new Epetra_Vector(x.Map());
    inargs.set_x_dot(Teuchos::rcp(x_dot, false));
  }

  // Create outargs
  EpetraExt::ModelEvaluator::OutArgs outargs = model_->createOutArgs();
  EpetraExt::ModelEvaluator::Evaluation<Epetra_Vector> eval_f;
  eval_f.reset(Teuchos::null, 
               EpetraExt::ModelEvaluator::EVAL_TYPE_VERY_APPROX_DERIV);
  outargs.set_f(eval_f);
  outargs.set_WPrec(Teuchos::rcp(&M, false));

  model_->evalModel(inargs, outargs);

  return true;
}
int main(int argc, char *argv[]) {

  int status=0; // 0 = pass, failures are incremented
  int overall_status=0; // 0 = pass, failures are incremented over multiple tests
  bool success=true;

  // Initialize MPI 
  Teuchos::GlobalMPISession mpiSession(&argc,&argv);
  int Proc=mpiSession.getRank();
#ifdef HAVE_MPI
  MPI_Comm appComm = MPI_COMM_WORLD;
#else
  int appComm=0;
#endif

  using Teuchos::RCP;
  using Teuchos::rcp;
  std::string inputFile;

  bool doAll = (argc==1);
  if (argc>1) doAll = !strcmp(argv[1],"-v");

  for (int iTest=0; iTest<3; iTest++) {

    if (doAll) {
      switch (iTest) {
       case 0: inputFile="input_Solve_VV.xml"; break;
       case 1: inputFile="input_Solve_TR.xml"; break;
       case 2: inputFile="input_Solve_NB.xml"; break;
       default : std::cout << "iTest logic error " << std::endl; exit(-1);
      }
    }
     else {
      inputFile=argv[1];
      iTest = 999;
    }

    if (Proc==0)
     std::cout << "===================================================\n"
          << "======  Running input file: "<< inputFile <<"\n"
          << "===================================================\n"
          << std::endl;
    
    try {

      // Create (1) a Model Evaluator and (2) a ParameterList
      RCP<EpetraExt::ModelEvaluator> Model = rcp(new MockModelEval_B(appComm));

      RCP<Teuchos::ParameterList> piroParams =
         rcp(new Teuchos::ParameterList("Piro Parameters"));
      Teuchos::updateParametersFromXmlFile(inputFile, piroParams.ptr());

      // Use these two objects to construct a Piro solved application 
      //   EpetraExt::ModelEvaluator is  base class of all Piro::Epetra solvers
      RCP<EpetraExt::ModelEvaluator> piro;

      std::string& solver = piroParams->get("Solver Type","NOX");
      RCP<NOX::Epetra::Observer> observer = rcp(new ObserveSolution_Epetra());

      if (solver=="NOX") {
        piro = rcp(new Piro::Epetra::NOXSolver(piroParams, Model, observer));
      }
      else if (solver=="Velocity Verlet") {
        piro = rcp(new Piro::Epetra::VelocityVerletSolver(
                       piroParams, Model, observer));
      }
      else if (solver=="Trapezoid Rule") {
        piro = rcp(new Piro::Epetra::TrapezoidRuleSolver(
                       piroParams, Model, observer));
      }
      else if (solver=="Newmark") {
        piro = rcp(new Piro::Epetra::NewmarkSolver(
                       piroParams, Model, observer));
      }
      else
        TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
          "Error: Unknown Piro Solver : " << solver);

      // Now the (somewhat cumbersome) setting of inputs and outputs
      EpetraExt::ModelEvaluator::InArgs inArgs = piro->createInArgs();
      TEUCHOS_ASSERT(inArgs.Np() > 0); // Number of *vectors* of parameters
      RCP<Epetra_Vector> p1 = rcp(new Epetra_Vector(*(piro->get_p_init(0))));
      inArgs.set_p(0,p1);

      // Set output arguments to evalModel call
      EpetraExt::ModelEvaluator::OutArgs outArgs = piro->createOutArgs();
      TEUCHOS_ASSERT(outArgs.Ng() >= 2); // Number of *vectors* of responses
      RCP<Epetra_Vector> g1 = rcp(new Epetra_Vector(*(piro->get_g_map(0))));
      outArgs.set_g(0,g1);
      // Solution vector is returned as extra respons vector
      RCP<Epetra_Vector> gx = rcp(new Epetra_Vector(*(piro->get_g_map(1))));
      outArgs.set_g(1,gx);

      // Now, solve the problem and return the responses
      piro->evalModel(inArgs, outArgs);

      // Print out everything
      if (Proc == 0)
        std::cout << "Finished Model Evaluation: Printing everything {Exact in brackets}" 
             << "\n-----------------------------------------------------------------"
             << std::setprecision(9) << std::endl;

      p1->Print(std::cout << "\nParameters! {1}\n");
      g1->Print(std::cout << "\nResponses! {0.0}\n");
      gx->Print(std::cout << "\nSolution! {0.0}\n");

      if (Proc == 0)
        std::cout <<
          "\n-----------------------------------------------------------------\n";
    }
    TEUCHOS_STANDARD_CATCH_STATEMENTS(true, std::cerr, success);
    if (!success) status=10; else status=0;

    overall_status += status;

  }

  if (Proc==0) {
    if (overall_status==0) std::cout << "\nTEST PASSED\n" << std::endl;
    else std::cout << "\nTEST Failed:  " << overall_status << std::endl;
  }

  return status;
}
Example #8
0
int main(int argc, char *argv[]) {

  int status=0; // 0 = pass, failures are incremented

  // Initialize MPI and timer
  int Proc=0;
#ifdef HAVE_MPI
  MPI_Init(&argc,&argv);
  double total_time = -MPI_Wtime();
  (void) MPI_Comm_rank(MPI_COMM_WORLD, &Proc);
  MPI_Comm appComm = MPI_COMM_WORLD;
#else
  int appComm=0;
#endif

  using Teuchos::RCP;
  using Teuchos::rcp;

  // Command-line argument for input file
  //char* defaultfile="input_1.xml";
  
  try {

    RCP<EpetraExt::ModelEvaluator> Model = rcp(new MockModelEval_A(appComm));

    // Set input arguments to evalModel call
    EpetraExt::ModelEvaluator::InArgs inArgs = Model->createInArgs();

    RCP<Epetra_Vector> x = rcp(new Epetra_Vector(*(Model->get_x_init())));
    inArgs.set_x(x);

    int num_p = inArgs.Np();     // Number of *vectors* of parameters
    RCP<Epetra_Vector> p1;
    if (num_p > 0) {
      p1 = rcp(new Epetra_Vector(*(Model->get_p_init(0))));
      inArgs.set_p(0,p1);
    }
    int numParams = p1->MyLength(); // Number of parameters in p1 vector

    // Set output arguments to evalModel call
    EpetraExt::ModelEvaluator::OutArgs outArgs = Model->createOutArgs();

    RCP<Epetra_Vector> f = rcp(new Epetra_Vector(x->Map()));
    outArgs.set_f(f);

    int num_g = outArgs.Ng(); // Number of *vectors* of responses
    RCP<Epetra_Vector> g1;
    if (num_g > 0) {
      g1 = rcp(new Epetra_Vector(*(Model->get_g_map(0))));
      outArgs.set_g(0,g1);
    }

    RCP<Epetra_Operator> W_op = Model->create_W();
    outArgs.set_W(W_op);

    RCP<Epetra_MultiVector> dfdp = rcp(new Epetra_MultiVector(
                             *(Model->get_x_map()), numParams));
    outArgs.set_DfDp(0, dfdp);

    RCP<Epetra_MultiVector> dgdp = rcp(new Epetra_MultiVector(g1->Map(), numParams));
    outArgs.set_DgDp(0, 0, dgdp);

    RCP<Epetra_MultiVector> dgdx = rcp(new Epetra_MultiVector(x->Map(), g1->MyLength()));
    outArgs.set_DgDx(0, dgdx);

    // Now, evaluate the model!
    Model->evalModel(inArgs, outArgs);

    // Print out everything
   if (Proc == 0)
    cout << "Finished Model Evaluation: Printing everything {Exact in brackets}" 
         << "\n-----------------------------------------------------------------"
         << std::setprecision(9) << endl;
    x->Print(cout << "\nSolution vector! {3,3,3,3}\n");
    if (num_p>0) p1->Print(cout << "\nParameters! {1,1}\n");
    f->Print(cout << "\nResidual! {8,5,0,-7}\n");
    if (num_g>0) g1->Print(cout << "\nResponses! {2}\n");
    RCP<Epetra_CrsMatrix> W = Teuchos::rcp_dynamic_cast<Epetra_CrsMatrix>(W_op, true);
    W->Print(cout << "\nJacobian! {6 on diags}\n");
    dfdp->Print(cout << "\nDfDp sensitivity MultiVector! {-1,0,0,0}{0,-4,-6,-8}\n");
    dgdp->Print(cout << "\nDgDp response sensitivity MultiVector!{2,2}\n");
    dgdx->Print(cout << "\nDgDx^T response gradient MultiVector! {-2,-2,-2,-2}\n");

    if (Proc == 0)
     cout <<
      "\n-----------------------------------------------------------------\n";
  }

  catch (std::exception& e) {
    cout << e.what() << endl;
    status = 10;
  }
  catch (string& s) {
    cout << s << endl;
    status = 20;
  }
  catch (char *s) {
    cout << s << endl;
    status = 30;
  }
  catch (...) {
    cout << "Caught unknown exception!" << endl;
    status = 40;
  }

#ifdef HAVE_MPI
  total_time +=  MPI_Wtime();
  MPI_Barrier(MPI_COMM_WORLD);
  if (Proc==0) cout << "\n\nTOTAL TIME     " 
                    << total_time << endl;
  MPI_Finalize() ;
#endif

  if (Proc==0) {
    if (status==0) 
      cout << "TEST PASSED" << endl;
    else 
      cout << "TEST Failed" << endl;
  }

  return status;
}
void EpetraExt::MultiPointModelEvaluator::evalModel( const InArgs& inArgs,
                                            const OutArgs& outArgs ) const
{

  EpetraExt::ModelEvaluator::InArgs  underlyingInArgs  = underlyingME->createInArgs();
  EpetraExt::ModelEvaluator::OutArgs underlyingOutArgs = underlyingME->createOutArgs();

  //temp code for multipoint param q vec
/*
  Teuchos::RefCountPtr<Epetra_Vector> q =
    Teuchos::rcp(new Epetra_Vector(*(underlyingME->get_p_map(1))));
*/

  // Parse InArgs
  Teuchos::RefCountPtr<const Epetra_Vector> p_in = inArgs.get_p(0);
  if (p_in.get()) underlyingInArgs.set_p(0, p_in);

  Teuchos::RefCountPtr<const Epetra_Vector> x_in = inArgs.get_x();
  block_x->Epetra_Vector::operator=(*x_in); //copy into block vector

  // Parse OutArgs
  Teuchos::RefCountPtr<Epetra_Vector> f_out = outArgs.get_f();

  Teuchos::RefCountPtr<Epetra_Operator> W_out = outArgs.get_W();
  Teuchos::RefCountPtr<EpetraExt::BlockCrsMatrix> W_block =
     Teuchos::rcp_dynamic_cast<EpetraExt::BlockCrsMatrix>(W_out);

  Teuchos::RefCountPtr<Epetra_Vector> g_out;
  if (underlyingNg) g_out = outArgs.get_g(0);
  if (g_out.get()) g_out->PutScalar(0.0);

  EpetraExt::ModelEvaluator::Derivative DfDp_out = outArgs.get_DfDp(0);

  EpetraExt::ModelEvaluator::Derivative DgDx_out;
  EpetraExt::ModelEvaluator::Derivative DgDp_out;
  if (underlyingNg) {
    DgDx_out = outArgs.get_DgDx(0);
    DgDp_out = outArgs.get_DgDp(0,0);
    if (!DgDx_out.isEmpty()) DgDx_out.getMultiVector()->PutScalar(0.0);
    if (!DgDp_out.isEmpty()) DgDp_out.getMultiVector()->PutScalar(0.0);
  }

  // For mathcingProblems, g is needed to calc DgDx DgDp, so ask for
  //  g even if it isn't requested.
  bool need_g = g_out.get();
  if (matchingProblem)
    if ( !DgDx_out.isEmpty() || !DgDp_out.isEmpty() ) need_g = true;


  // Begin loop over Points (steps) owned on this proc
  for (int i=0; i < timeStepsOnTimeDomain; i++) {

    // Set MultiPoint parameter vector
    underlyingInArgs.set_p(1, (*q_vec)[i]);

    // Set InArgs
    if(longlong) {
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
      block_x->ExtractBlockValues(*split_x, (*rowIndex_LL)[i]);
#endif
    }
    else {
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
      block_x->ExtractBlockValues(*split_x, (*rowIndex_int)[i]);
#endif
    }
    underlyingInArgs.set_x(split_x);

    // Set OutArgs
    if (f_out.get()) underlyingOutArgs.set_f(split_f);

    if (need_g) underlyingOutArgs.set_g(0, split_g);

    if (W_out.get()) underlyingOutArgs.set_W(split_W);

    if (!DfDp_out.isEmpty()) underlyingOutArgs.set_DfDp(0, *deriv_DfDp);

    if (!DgDx_out.isEmpty()) underlyingOutArgs.set_DgDx(0, *deriv_DgDx);

    if (!DgDp_out.isEmpty()) underlyingOutArgs.set_DgDp(0, 0, *deriv_DgDp);

    //********Eval Model ********/
    underlyingME->evalModel(underlyingInArgs, underlyingOutArgs);
    //********Eval Model ********/

    // If matchingProblem, modify all g-related quantitites G = 0.5*(g-g*)^2 / g*^2
    if (matchingProblem) {
      if (need_g) {
        double diff = (*split_g)[0] -  (*(*matching_vec)[i])[0];
        double nrmlz = fabs((*(*matching_vec)[i])[0]) + 1.0e-6;
        (*split_g)[0] = 0.5 * diff * diff/(nrmlz*nrmlz);
        if (!DgDx_out.isEmpty()) split_DgDx->Scale(diff/(nrmlz*nrmlz));
        if (!DgDp_out.isEmpty()) split_DgDp->Scale(diff/(nrmlz*nrmlz));
      }
    }

    // Repackage block components into global block matrx/vector/multivector
    if(longlong) {
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
      if (f_out.get()) block_f->LoadBlockValues(*split_f, (*rowIndex_LL)[i]);
      if (W_out.get()) W_block->LoadBlock(*split_W, i, 0);
        // note: split_DfDp points inside deriv_DfDp
      if (!DfDp_out.isEmpty()) block_DfDp->LoadBlockValues(*split_DfDp, (*rowIndex_LL)[i]);
      if (!DgDx_out.isEmpty()) block_DgDx->LoadBlockValues(*split_DgDx, (*rowIndex_LL)[i]);
#endif
    }
    else {
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
      if (f_out.get()) block_f->LoadBlockValues(*split_f, (*rowIndex_int)[i]);
      if (W_out.get()) W_block->LoadBlock(*split_W, i, 0);
        // note: split_DfDp points inside deriv_DfDp
      if (!DfDp_out.isEmpty()) block_DfDp->LoadBlockValues(*split_DfDp, (*rowIndex_int)[i]);
      if (!DgDx_out.isEmpty()) block_DgDx->LoadBlockValues(*split_DgDx, (*rowIndex_int)[i]);
#endif
    }

    // Assemble multiple steps on this domain into g and dgdp(0) vectors
    if (g_out.get()) g_out->Update(1.0, *split_g, 1.0);

    if (!DgDp_out.isEmpty())
      DgDp_out.getMultiVector()->Update(1.0, *split_DgDp, 1.0);

  } // End loop over multiPoint steps on this domain/cluster

  //Copy block vectors into *_out vectors of same size
  if (f_out.get()) f_out->operator=(*block_f);
  if (!DfDp_out.isEmpty())
    DfDp_out.getMultiVector()->operator=(*block_DfDp);
  if (!DgDx_out.isEmpty())
    DgDx_out.getMultiVector()->operator=(*block_DgDx);

  //Sum together obj fn contributions from differnt Domains (clusters).
  if (numTimeDomains > 1) {
    double factorToZeroOutCopies = 0.0;
    if (globalComm->SubDomainComm().MyPID()==0) factorToZeroOutCopies = 1.0;
    if (g_out.get()) {
      (*g_out).Scale(factorToZeroOutCopies);
      double* vPtr = &((*g_out)[0]);
      Epetra_Vector tmp = *(g_out.get());
      globalComm->SumAll( &(tmp[0]), vPtr, num_g0);
    }
    if (!DgDp_out.isEmpty()) {
      DgDp_out.getMultiVector()->Scale(factorToZeroOutCopies);
      double* mvPtr = (*DgDp_out.getMultiVector())[0];
      Epetra_MultiVector tmp = *(DgDp_out.getMultiVector());
      globalComm->SumAll(tmp[0], mvPtr, num_dg0dp0);
    }
  }
}