예제 #1
0
void
FullSolveMultiApp::solveStep(Real /*dt*/, Real /*target_time*/, bool auto_advance)
{
  if (!auto_advance)
    mooseError("FullSolveMultiApp is not compatible with auto_advance=false");

  if (!_has_an_app)
    return;

  if (_solved)
    return;

  _console << "Fully Solving MultiApp " << name() << std::endl;

  MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);

  int rank;
  int ierr;
  ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);

  for (unsigned int i=0; i<_my_num_apps; i++)
  {
    Executioner * ex = _executioners[i];
    ex->execute();
  }

  // Swap back
  Moose::swapLibMeshComm(swapped);

  _solved = true;

  _console << "Finished Solving MultiApp " << name() << std::endl;
}
예제 #2
0
파일: Moose.C 프로젝트: atomica/moose
MPI_Comm
swapLibMeshComm(MPI_Comm new_comm)
{
  int ierr;

  MPI_Comm old_comm = libMesh::COMM_WORLD;
  libMesh::COMM_WORLD = new_comm;
  libMesh::CommWorld = new_comm;
#ifdef LIBMESH_HAVE_PETSC
  PETSC_COMM_WORLD = new_comm;
#endif //LIBMESH_HAVE_PETSC

  int pid;
  ierr = MPI_Comm_rank(new_comm, &pid); mooseCheckMPIErr(ierr);

  int n_procs;
  ierr = MPI_Comm_size(new_comm, &n_procs); mooseCheckMPIErr(ierr);

  libMesh::libMeshPrivateData::_processor_id = pid;
  libMesh::libMeshPrivateData::_n_processors = n_procs;

  return old_comm;
}
예제 #3
0
void
TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
{
  if (_sub_cycling && !auto_advance)
    mooseError("TransientMultiApp with sub_cycling=true is not compatible with auto_advance=false");

  if (_catch_up && !auto_advance)
    mooseError("TransientMultiApp with catch_up=true is not compatible with auto_advance=false");

  if (!_has_an_app)
    return;

  _auto_advance = auto_advance;

  Moose::out << "Solving MultiApp " << _name << std::endl;

// "target_time" must always be in global time
  target_time += _app.getGlobalTimeOffset();

  MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);

  int rank;
  int ierr;
  ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);

  for (unsigned int i=0; i<_my_num_apps; i++)
  {

    FEProblem * problem = appProblem(_first_local_app + i);
    OutputWarehouse & output_warehouse = _apps[i]->getOutputWarehouse();

    Transient * ex = _transient_executioners[i];

    // The App might have a different local time from the rest of the problem
    Real app_time_offset = _apps[i]->getGlobalTimeOffset();

    if ((ex->getTime() + app_time_offset) + 2e-14 >= target_time) // Maybe this MultiApp was already solved
      continue;

    if (_sub_cycling)
    {
      Real time_old = ex->getTime() + app_time_offset;

      if (_interpolate_transfers)
      {
        AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
        System & libmesh_aux_system = aux_system.system();

        NumericVector<Number> & solution = *libmesh_aux_system.solution;
        NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");

        solution.close();

        // Save off the current auxiliary solution
        transfer_old = solution;

        transfer_old.close();

        // Snag all of the local dof indices for all of these variables
        AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
        ConstElemRange & elem_range = *problem->mesh().getActiveLocalElementRange();
        Threads::parallel_reduce(elem_range, aldit);

        _transferred_dofs = aldit._all_dof_indices;
      }

      if (_output_sub_cycles)
        output_warehouse.allowOutput(true);
      else
        output_warehouse.allowOutput(false);

      ex->setTargetTime(target_time-app_time_offset);

//      unsigned int failures = 0;

      bool at_steady = false;

      // Now do all of the solves we need
      while(true)
      {
        if (_first != true)
          ex->incrementStepOrReject();
        _first = false;

        if (!(!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time))
          break;

        ex->computeDT();

        if (_interpolate_transfers)
        {
          // See what time this executioner is going to go to.
          Real future_time = ex->getTime() + app_time_offset + ex->getDT();

          // How far along we are towards the target time:
          Real step_percent = (future_time - time_old) / (target_time - time_old);

          Real one_minus_step_percent = 1.0 - step_percent;

          // Do the interpolation for each variable that was transferred to
          FEProblem * problem = appProblem(_first_local_app + i);
          AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
          System & libmesh_aux_system = aux_system.system();

          NumericVector<Number> & solution = *libmesh_aux_system.solution;
          NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
          NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");

          solution.close(); // Just to be sure
          transfer.close();
          transfer_old.close();

          std::set<dof_id_type>::iterator it  = _transferred_dofs.begin();
          std::set<dof_id_type>::iterator end = _transferred_dofs.end();

          for(; it != end; ++it)
          {
            dof_id_type dof = *it;
            solution.set(dof, (transfer_old(dof) * one_minus_step_percent) + (transfer(dof) * step_percent));
//            solution.set(dof, transfer_old(dof));
//            solution.set(dof, transfer(dof));
//            solution.set(dof, 1);
          }

          solution.close();
        }

        ex->takeStep();

        bool converged = ex->lastSolveConverged();

        if (!converged)
        {
          mooseWarning("While sub_cycling "<<_name<<_first_local_app+i<<" failed to converge!"<<std::endl);
          _failures++;

          if (_failures > _max_failures)
            mooseError("While sub_cycling "<<_name<<_first_local_app+i<<" REALLY failed!"<<std::endl);
        }

        Real solution_change_norm = ex->getSolutionChangeNorm();

        if (_detect_steady_state)
          Moose::out << "Solution change norm: " << solution_change_norm << std::endl;

        if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
        {
          Moose::out << "Detected Steady State!  Fast-forwarding to " << target_time << std::endl;

          at_steady = true;

         // Indicate that the next output call (occurs in ex->endStep()) should output, regarless of intervals etc...
          output_warehouse.forceOutput();

          // Clean up the end
          ex->endStep(target_time-app_time_offset);
        }
        else
          ex->endStep();
      }

      // If we were looking for a steady state, but didn't reach one, we still need to output one more time
      if (!at_steady)
      {
        output_warehouse.forceOutput();
        output_warehouse.outputStep();
     }

    }
    else if (_tolerate_failure)
    {
      ex->takeStep(dt);
      output_warehouse.forceOutput();
      ex->endStep(target_time-app_time_offset);
    }
    else
    {
      Moose::out << "Solving Normal Step!" << std::endl;
      if (auto_advance)
        if (_first != true)
          ex->incrementStepOrReject();

      if (auto_advance)
        output_warehouse.allowOutput(true);

      ex->takeStep(dt);

      if (auto_advance)
      {
        ex->endStep();

        if (!ex->lastSolveConverged())
        {
          mooseWarning(_name << _first_local_app+i << " failed to converge!" << std::endl);

          if (_catch_up)
          {
            Moose::out << "Starting Catch Up!" << std::endl;

            bool caught_up = false;

            unsigned int catch_up_step = 0;

            Real catch_up_dt = dt/2;

            while(!caught_up && catch_up_step < _max_catch_up_steps)
            {
              Moose::err << "Solving " << _name << "catch up step " << catch_up_step << std::endl;
              ex->incrementStepOrReject();

              ex->computeDT();
              ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves

              if (ex->lastSolveConverged())
              {
                if (ex->getTime() + app_time_offset + ex->timestepTol()*std::abs(ex->getTime()) >= target_time)
                {
                  output_warehouse.forceOutput();
                  output_warehouse.outputStep();
                  caught_up = true;
                }
              }
              else
                catch_up_dt /= 2.0;

              ex->endStep();

              catch_up_step++;
            }

            if (!caught_up)
              mooseError(_name << " Failed to catch up!\n");

            output_warehouse.allowOutput(true);
           }
        }
      }
    }
  }

  _first = false;

  // Swap back
  Moose::swapLibMeshComm(swapped);

  _transferred_vars.clear();

  Moose::out << "Finished Solving MultiApp " << _name << std::endl;
}
예제 #4
0
파일: MultiApp.C 프로젝트: beelbw/moose
void
MultiApp::buildComm()
{
  int ierr;

  ierr = MPI_Comm_size(_orig_comm, &_orig_num_procs); mooseCheckMPIErr(ierr);
  ierr = MPI_Comm_rank(_orig_comm, &_orig_rank); mooseCheckMPIErr(ierr);

  struct utsname sysInfo;
  uname(&sysInfo);

  _node_name = sysInfo.nodename;

  // If we have more apps than processors then we're just going to divide up the work
  if (_total_num_apps >= (unsigned)_orig_num_procs)
  {
    _my_comm = MPI_COMM_SELF;
    _my_rank = 0;

    _my_num_apps = _total_num_apps/_orig_num_procs;
    unsigned int jobs_left = _total_num_apps - (_my_num_apps * _orig_num_procs);

    if (jobs_left != 0)
    {
      // Spread the remaining jobs out over the first set of processors
      if ((unsigned)_orig_rank < jobs_left)  // (these are the "jobs_left_pids" ie the pids that are snatching up extra jobs)
      {
        _my_num_apps += 1;
        _first_local_app = _my_num_apps * _orig_rank;
      }
      else
      {
        unsigned int num_apps_in_jobs_left_pids = (_my_num_apps + 1) * jobs_left;
        unsigned int distance_to_jobs_left_pids = _orig_rank - jobs_left;

        _first_local_app = num_apps_in_jobs_left_pids + (_my_num_apps * distance_to_jobs_left_pids);
      }
    }
    else
      _first_local_app = _my_num_apps * _orig_rank;

    return;
  }

  // In this case we need to divide up the processors that are going to work on each app
  int rank;
  ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);

  unsigned int procs_per_app = _orig_num_procs / _total_num_apps;

  if (_max_procs_per_app < procs_per_app)
    procs_per_app = _max_procs_per_app;

  int my_app = rank / procs_per_app;
  unsigned int procs_for_my_app = procs_per_app;

  if ((unsigned int) my_app > _total_num_apps-1 && procs_for_my_app == _max_procs_per_app)
  {
    // If we've already hit the max number of procs per app then this processor
    // won't have an app at all
    _my_num_apps = 0;
    _has_an_app = false;
  }
  else if ((unsigned int) my_app >= _total_num_apps-1) // The last app will gain any left-over procs
  {
    my_app = _total_num_apps - 1;
//    procs_for_my_app += _orig_num_procs % _total_num_apps;
    _first_local_app = my_app;
    _my_num_apps = 1;
  }
  else
  {
    _first_local_app = my_app;
    _my_num_apps = 1;
  }

  if (_has_an_app)
  {
    ierr = MPI_Comm_split(_orig_comm, _first_local_app, rank, &_my_comm); mooseCheckMPIErr(ierr);
    ierr = MPI_Comm_rank(_my_comm, &_my_rank); mooseCheckMPIErr(ierr);
  }
  else
  {
    ierr = MPI_Comm_split(_orig_comm, MPI_UNDEFINED, rank, &_my_comm); mooseCheckMPIErr(ierr);
    _my_rank = 0;
  }
}
예제 #5
0
bool
TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
{
  if (!_has_an_app)
    return true;

  _auto_advance = auto_advance;

  _console << "Solving MultiApp " << name() << std::endl;

  // "target_time" must always be in global time
  target_time += _app.getGlobalTimeOffset();

  Moose::ScopedCommSwapper swapper(_my_comm);
  bool return_value = true;

  // Make sure we swap back the communicator regardless of how this routine is exited
  try
  {
    int rank;
    int ierr;
    ierr = MPI_Comm_rank(_orig_comm, &rank);
    mooseCheckMPIErr(ierr);

    for (unsigned int i = 0; i < _my_num_apps; i++)
    {

      FEProblemBase & problem = appProblemBase(_first_local_app + i);

      Transient * ex = _transient_executioners[i];

      // The App might have a different local time from the rest of the problem
      Real app_time_offset = _apps[i]->getGlobalTimeOffset();

      // Maybe this MultiApp was already solved
      if ((ex->getTime() + app_time_offset + 2e-14 >= target_time) ||
          (ex->getTime() >= ex->endTime()))
        continue;

      if (_sub_cycling)
      {
        Real time_old = ex->getTime() + app_time_offset;

        if (_interpolate_transfers)
        {
          AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
          System & libmesh_aux_system = aux_system.system();

          NumericVector<Number> & solution = *libmesh_aux_system.solution;
          NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");

          solution.close();

          // Save off the current auxiliary solution
          transfer_old = solution;

          transfer_old.close();

          // Snag all of the local dof indices for all of these variables
          AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
          ConstElemRange & elem_range = *problem.mesh().getActiveLocalElementRange();
          Threads::parallel_reduce(elem_range, aldit);

          _transferred_dofs = aldit._all_dof_indices;
        }

        // Disable/enable output for sub cycling
        problem.allowOutput(_output_sub_cycles);         // disables all outputs, including console
        problem.allowOutput<Console>(_print_sub_cycles); // re-enables Console to print, if desired

        ex->setTargetTime(target_time - app_time_offset);

        //      unsigned int failures = 0;

        bool at_steady = false;

        if (_first && !_app.isRecovering())
          problem.advanceState();

        bool local_first = _first;

        // Now do all of the solves we need
        while ((!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time) ||
               !ex->lastSolveConverged())
        {
          if (local_first != true)
            ex->incrementStepOrReject();

          local_first = false;

          ex->preStep();
          ex->computeDT();

          if (_interpolate_transfers)
          {
            // See what time this executioner is going to go to.
            Real future_time = ex->getTime() + app_time_offset + ex->getDT();

            // How far along we are towards the target time:
            Real step_percent = (future_time - time_old) / (target_time - time_old);

            Real one_minus_step_percent = 1.0 - step_percent;

            // Do the interpolation for each variable that was transferred to
            FEProblemBase & problem = appProblemBase(_first_local_app + i);
            AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
            System & libmesh_aux_system = aux_system.system();

            NumericVector<Number> & solution = *libmesh_aux_system.solution;
            NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
            NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");

            solution.close(); // Just to be sure
            transfer.close();
            transfer_old.close();

            for (const auto & dof : _transferred_dofs)
            {
              solution.set(dof,
                           (transfer_old(dof) * one_minus_step_percent) +
                               (transfer(dof) * step_percent));
              //            solution.set(dof, transfer_old(dof));
              //            solution.set(dof, transfer(dof));
              //            solution.set(dof, 1);
            }

            solution.close();
          }

          ex->takeStep();

          bool converged = ex->lastSolveConverged();

          if (!converged)
          {
            mooseWarning(
                "While sub_cycling ", name(), _first_local_app + i, " failed to converge!\n");

            _failures++;

            if (_failures > _max_failures)
            {
              std::stringstream oss;
              oss << "While sub_cycling " << name() << _first_local_app << i << " REALLY failed!";
              throw MultiAppSolveFailure(oss.str());
            }
          }

          Real solution_change_norm = ex->getSolutionChangeNorm();

          if (_detect_steady_state)
            _console << "Solution change norm: " << solution_change_norm << std::endl;

          if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
          {
            _console << "Detected Steady State!  Fast-forwarding to " << target_time << std::endl;

            at_steady = true;

            // Indicate that the next output call (occurs in ex->endStep()) should output,
            // regardless of intervals etc...
            problem.forceOutput();

            // Clean up the end
            ex->endStep(target_time - app_time_offset);
            ex->postStep();
          }
          else
          {
            ex->endStep();
            ex->postStep();
          }
        }

        // If we were looking for a steady state, but didn't reach one, we still need to output one
        // more time, regardless of interval
        if (!at_steady)
          problem.outputStep(EXEC_FORCED);

      } // sub_cycling
      else if (_tolerate_failure)
      {
        ex->takeStep(dt);
        ex->endStep(target_time - app_time_offset);
        ex->postStep();
      }
      else
      {
        _console << "Solving Normal Step!" << std::endl;

        if (_first && !_app.isRecovering())
          problem.advanceState();

        if (auto_advance)
          problem.allowOutput(true);

        ex->takeStep(dt);

        if (auto_advance)
        {
          ex->endStep();
          ex->postStep();

          if (!ex->lastSolveConverged())
          {
            mooseWarning(name(), _first_local_app + i, " failed to converge!\n");

            if (_catch_up)
            {
              _console << "Starting Catch Up!" << std::endl;

              bool caught_up = false;

              unsigned int catch_up_step = 0;

              Real catch_up_dt = dt / 2;

              while (!caught_up && catch_up_step < _max_catch_up_steps)
              {
                _console << "Solving " << name() << " catch up step " << catch_up_step << std::endl;
                ex->incrementStepOrReject();

                ex->computeDT();
                ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
                ex->endStep();

                if (ex->lastSolveConverged())
                {
                  if (ex->getTime() + app_time_offset +
                          (ex->timestepTol() * std::abs(ex->getTime())) >=
                      target_time)
                  {
                    problem.outputStep(EXEC_FORCED);
                    caught_up = true;
                  }
                }
                else
                  catch_up_dt /= 2.0;

                ex->postStep();

                catch_up_step++;
              }

              if (!caught_up)
                throw MultiAppSolveFailure(name() + " Failed to catch up!\n");
            }
          }
        }
        else
        {
          if (!ex->lastSolveConverged())
          {
            // Even if we don't allow auto_advance - we can still catch up to the current time if
            // possible
            if (_catch_up)
            {
              _console << "Starting Catch Up!" << std::endl;

              bool caught_up = false;

              unsigned int catch_up_step = 0;

              Real catch_up_dt = dt / 2;

              // Note: this loop will _break_ if target_time is satisfied
              while (catch_up_step < _max_catch_up_steps)
              {
                _console << "Solving " << name() << " catch up step " << catch_up_step << std::endl;
                ex->incrementStepOrReject();

                ex->computeDT();
                ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves

                // This is required because we can't call endStep() yet
                // (which normally increments time)
                Real current_time = ex->getTime() + ex->getDT();

                if (ex->lastSolveConverged())
                {
                  if (current_time + app_time_offset +
                          (ex->timestepTol() * std::abs(current_time)) >=
                      target_time)
                  {
                    caught_up = true;
                    break; // break here so that we don't run endStep() or postStep() since this
                           // MultiApp should NOT be auto_advanced
                  }
                }
                else
                  catch_up_dt /= 2.0;

                ex->endStep();
                ex->postStep();

                catch_up_step++;
              }

              if (!caught_up)
                throw MultiAppSolveFailure(name() + " Failed to catch up!\n");
            }
            else
              throw MultiAppSolveFailure(name() + " failed to converge");
          }
        }
      }

      // Re-enable all output (it may of been disabled by sub-cycling)
      problem.allowOutput(true);
    }

    _first = false;

    _console << "Successfully Solved MultiApp " << name() << "." << std::endl;
  }
  catch (MultiAppSolveFailure & e)
  {
    mooseWarning(e.what());
    _console << "Failed to Solve MultiApp " << name() << ", attempting to recover." << std::endl;
    return_value = false;
  }

  _transferred_vars.clear();

  return return_value;
}