Example #1
0
int main(int argc, char *argv[]) {
  
  bool StopCalc = false;
  double StartTime = 0.0, StopTime = 0.0, UsedTime = 0.0;
  unsigned long ExtIter = 0;
  unsigned short iMesh, iZone, iSol, nZone, nDim;
  char config_file_name[MAX_STRING_SIZE];
  char runtime_file_name[MAX_STRING_SIZE];
  ofstream ConvHist_file;
  int rank = MASTER_NODE;
  int size = SINGLE_NODE;
  
  /*--- MPI initialization, and buffer setting ---*/
  
#ifdef HAVE_MPI
  int *bptr, bl;
  MPI_Init(&argc, &argv);
  MPI_Buffer_attach( malloc(BUFSIZE), BUFSIZE );
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  MPI_Comm_size(MPI_COMM_WORLD, &size);
#endif
  
  /*--- Create pointers to all of the classes that may be used throughout
   the SU2_CFD code. In general, the pointers are instantiated down a
   heirarchy over all zones, multigrid levels, equation sets, and equation
   terms as described in the comments below. ---*/
  
  COutput *output                       = NULL;
  CIntegration ***integration_container = NULL;
  CGeometry ***geometry_container       = NULL;
  CSolver ****solver_container          = NULL;
  CNumerics *****numerics_container     = NULL;
  CConfig **config_container            = NULL;
  CSurfaceMovement **surface_movement   = NULL;
  CVolumetricMovement **grid_movement   = NULL;
  CFreeFormDefBox*** FFDBox             = NULL;
  
  /*--- Load in the number of zones and spatial dimensions in the mesh file (If no config
   file is specified, default.cfg is used) ---*/
  
  if (argc == 2) { strcpy(config_file_name, argv[1]); }
  else { strcpy(config_file_name, "default.cfg"); }
  
  /*--- Read the name and format of the input mesh file to get from the mesh
   file the number of zones and dimensions from the numerical grid (required
   for variables allocation)  ---*/
  
  CConfig *config = NULL;
  config = new CConfig(config_file_name, SU2_CFD);
  
  nZone = GetnZone(config->GetMesh_FileName(), config->GetMesh_FileFormat(), config);
  nDim  = GetnDim(config->GetMesh_FileName(), config->GetMesh_FileFormat());
  
  /*--- Definition and of the containers for all possible zones. ---*/
  
  solver_container      = new CSolver***[nZone];
  integration_container = new CIntegration**[nZone];
  numerics_container    = new CNumerics****[nZone];
  config_container      = new CConfig*[nZone];
  geometry_container    = new CGeometry**[nZone];
  surface_movement      = new CSurfaceMovement*[nZone];
  grid_movement         = new CVolumetricMovement*[nZone];
  FFDBox                = new CFreeFormDefBox**[nZone];
  
  for (iZone = 0; iZone < nZone; iZone++) {
    solver_container[iZone]       = NULL;
    integration_container[iZone]  = NULL;
    numerics_container[iZone]     = NULL;
    config_container[iZone]       = NULL;
    geometry_container[iZone]     = NULL;
    surface_movement[iZone]       = NULL;
    grid_movement[iZone]          = NULL;
    FFDBox[iZone]                 = NULL;
  }
  
  /*--- Loop over all zones to initialize the various classes. In most
   cases, nZone is equal to one. This represents the solution of a partial
   differential equation on a single block, unstructured mesh. ---*/
  
  for (iZone = 0; iZone < nZone; iZone++) {
    
    /*--- Definition of the configuration option class for all zones. In this
     constructor, the input configuration file is parsed and all options are
     read and stored. ---*/
    
    config_container[iZone] = new CConfig(config_file_name, SU2_CFD, iZone, nZone, nDim, VERB_HIGH);
    
    
    /*--- Definition of the geometry class to store the primal grid in the
     partitioning process. ---*/
    
    CGeometry *geometry_aux = NULL;
    
    /*--- All ranks process the grid and call ParMETIS for partitioning ---*/
    
    geometry_aux = new CPhysicalGeometry(config_container[iZone], iZone, nZone);
    
    /*--- Color the initial grid and set the send-receive domains (ParMETIS) ---*/
    
    geometry_aux->SetColorGrid_Parallel(config_container[iZone]);
    
    /*--- Allocate the memory of the current domain, and divide the grid 
     between the ranks. ---*/
    
    geometry_container[iZone] = new CGeometry *[config_container[iZone]->GetnMGLevels()+1];
    geometry_container[iZone][MESH_0] = new CPhysicalGeometry(geometry_aux, config_container[iZone], 1);
    
    /*--- Deallocate the memory of geometry_aux ---*/
    
    delete geometry_aux;
    
    /*--- Add the Send/Receive boundaries ---*/
    
    geometry_container[iZone][MESH_0]->SetSendReceive(config_container[iZone]);
    
    /*--- Add the Send/Receive boundaries ---*/
    
    geometry_container[iZone][MESH_0]->SetBoundaries(config_container[iZone]);
    
  }
  
  if (rank == MASTER_NODE)
    cout << endl <<"------------------------- Geometry Preprocessing ------------------------" << endl;
  
  /*--- Preprocessing of the geometry for all zones. In this routine, the edge-
   based data structure is constructed, i.e. node and cell neighbors are
   identified and linked, face areas and volumes of the dual mesh cells are
   computed, and the multigrid levels are created using an agglomeration procedure. ---*/
  
  Geometrical_Preprocessing(geometry_container, config_container, nZone);
  
  if (rank == MASTER_NODE)
    cout << endl <<"------------------------- Solver Preprocessing --------------------------" << endl;
  
  for (iZone = 0; iZone < nZone; iZone++) {
    
    /*--- Computation of wall distances for turbulence modeling ---*/
    
    if ( (config_container[iZone]->GetKind_Solver() == RANS) ||
        (config_container[iZone]->GetKind_Solver() == ADJ_RANS) )
      geometry_container[iZone][MESH_0]->ComputeWall_Distance(config_container[iZone]);
    
    /*--- Computation of positive surface area in the z-plane which is used for
     the calculation of force coefficient (non-dimensionalization). ---*/
    
    geometry_container[iZone][MESH_0]->SetPositive_ZArea(config_container[iZone]);
    
    /*--- Set the near-field, interface and actuator disk boundary conditions, if necessary. ---*/
    
    for (iMesh = 0; iMesh <= config_container[iZone]->GetnMGLevels(); iMesh++) {
      geometry_container[iZone][iMesh]->MatchNearField(config_container[iZone]);
      geometry_container[iZone][iMesh]->MatchInterface(config_container[iZone]);
      geometry_container[iZone][iMesh]->MatchActuator_Disk(config_container[iZone]);
    }
    
    /*--- Definition of the solver class: solver_container[#ZONES][#MG_GRIDS][#EQ_SYSTEMS].
     The solver classes are specific to a particular set of governing equations,
     and they contain the subroutines with instructions for computing each spatial
     term of the PDE, i.e. loops over the edges to compute convective and viscous
     fluxes, loops over the nodes to compute source terms, and routines for
     imposing various boundary condition type for the PDE. ---*/
    
    solver_container[iZone] = new CSolver** [config_container[iZone]->GetnMGLevels()+1];
    for (iMesh = 0; iMesh <= config_container[iZone]->GetnMGLevels(); iMesh++)
      solver_container[iZone][iMesh] = NULL;
    
    for (iMesh = 0; iMesh <= config_container[iZone]->GetnMGLevels(); iMesh++) {
      solver_container[iZone][iMesh] = new CSolver* [MAX_SOLS];
      for (iSol = 0; iSol < MAX_SOLS; iSol++)
        solver_container[iZone][iMesh][iSol] = NULL;
    }
    Solver_Preprocessing(solver_container[iZone], geometry_container[iZone],
                         config_container[iZone], iZone);
    
    if (rank == MASTER_NODE)
      cout << endl <<"----------------- Integration and Numerics Preprocessing ----------------" << endl;
    
    /*--- Definition of the integration class: integration_container[#ZONES][#EQ_SYSTEMS].
     The integration class orchestrates the execution of the spatial integration
     subroutines contained in the solver class (including multigrid) for computing
     the residual at each node, R(U) and then integrates the equations to a
     steady state or time-accurately. ---*/
    
    integration_container[iZone] = new CIntegration*[MAX_SOLS];
    Integration_Preprocessing(integration_container[iZone], geometry_container[iZone],
                              config_container[iZone], iZone);
    
    if (rank == MASTER_NODE) cout << "Integration Preprocessing." << endl;
    
    /*--- Definition of the numerical method class:
     numerics_container[#ZONES][#MG_GRIDS][#EQ_SYSTEMS][#EQ_TERMS].
     The numerics class contains the implementation of the numerical methods for
     evaluating convective or viscous fluxes between any two nodes in the edge-based
     data structure (centered, upwind, galerkin), as well as any source terms
     (piecewise constant reconstruction) evaluated in each dual mesh volume. ---*/
    
    numerics_container[iZone] = new CNumerics***[config_container[iZone]->GetnMGLevels()+1];
    Numerics_Preprocessing(numerics_container[iZone], solver_container[iZone],
                           geometry_container[iZone], config_container[iZone], iZone);
    
    if (rank == MASTER_NODE) cout << "Numerics Preprocessing." << endl;
    
    /*--- Instantiate the geometry movement classes for the solution of unsteady
     flows on dynamic meshes, including rigid mesh transformations, dynamically
     deforming meshes, and time-spectral preprocessing. ---*/
    
    if (config_container[iZone]->GetGrid_Movement()) {
      if (rank == MASTER_NODE)
        cout << "Setting dynamic mesh structure." << endl;
      grid_movement[iZone] = new CVolumetricMovement(geometry_container[iZone][MESH_0]);
      FFDBox[iZone] = new CFreeFormDefBox*[MAX_NUMBER_FFD];
      surface_movement[iZone] = new CSurfaceMovement();
      surface_movement[iZone]->CopyBoundary(geometry_container[iZone][MESH_0], config_container[iZone]);
      if (config_container[iZone]->GetUnsteady_Simulation() == TIME_SPECTRAL)
        SetGrid_Movement(geometry_container[iZone], surface_movement[iZone], grid_movement[iZone],
                         FFDBox[iZone], solver_container[iZone], config_container[iZone], iZone, 0, 0);
    }
    
  }
  
  /*--- For the time-spectral solver, set the grid node velocities. ---*/
  
  if (config_container[ZONE_0]->GetUnsteady_Simulation() == TIME_SPECTRAL)
    SetTimeSpectral_Velocities(geometry_container, config_container, nZone);
  
  /*--- Coupling between zones (limited to two zones at the moment) ---*/
  
  if (nZone == 2) {
    if (rank == MASTER_NODE)
      cout << endl <<"--------------------- Setting Coupling Between Zones --------------------" << endl;
    geometry_container[ZONE_0][MESH_0]->MatchZone(config_container[ZONE_0], geometry_container[ZONE_1][MESH_0],
                                                  config_container[ZONE_1], ZONE_0, nZone);
    geometry_container[ZONE_1][MESH_0]->MatchZone(config_container[ZONE_1], geometry_container[ZONE_0][MESH_0],
                                                  config_container[ZONE_0], ZONE_1, nZone);
  }
  
  /*--- Definition of the output class (one for all zones). The output class
   manages the writing of all restart, volume solution, surface solution,
   surface comma-separated value, and convergence history files (both in serial
   and in parallel). ---*/
  
  output = new COutput();
  
  /*--- Open the convergence history file ---*/
  
  if (rank == MASTER_NODE)
    output->SetConvHistory_Header(&ConvHist_file, config_container[ZONE_0]);
  
  /*--- Check for an unsteady restart. Update ExtIter if necessary. ---*/
  if (config_container[ZONE_0]->GetWrt_Unsteady() && config_container[ZONE_0]->GetRestart())
    ExtIter = config_container[ZONE_0]->GetUnst_RestartIter();
  
  /*--- Main external loop of the solver. Within this loop, each iteration ---*/
  
  if (rank == MASTER_NODE)
    cout << endl <<"------------------------------ Begin Solver -----------------------------" << endl;
  
  /*--- Set up a timer for performance benchmarking (preprocessing time is not included) ---*/
  
#ifndef HAVE_MPI
  StartTime = double(clock())/double(CLOCKS_PER_SEC);
#else
  StartTime = MPI_Wtime();
#endif
  
  while (ExtIter < config_container[ZONE_0]->GetnExtIter()) {
    
    /*--- Set the value of the external iteration. ---*/
    
    config_container[ZONE_0]->SetExtIter(ExtIter);
    
    /*--- Read the target pressure ---*/
    
    if (config_container[ZONE_0]->GetInvDesign_Cp() == YES)
      output->SetCp_InverseDesign(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                  geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
    
    /*--- Read the target heat flux ---*/
    
    if (config_container[ZONE_0]->GetInvDesign_HeatFlux() == YES)
      output->SetHeat_InverseDesign(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                    geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
    
    /*--- Perform a single iteration of the chosen PDE solver. ---*/
    
    switch (config_container[ZONE_0]->GetKind_Solver()) {
        
      case EULER: case NAVIER_STOKES: case RANS:
        MeanFlowIteration(output, integration_container, geometry_container,
                          solver_container, numerics_container, config_container,
                          surface_movement, grid_movement, FFDBox);
        break;
        
      case TNE2_EULER: case TNE2_NAVIER_STOKES:
        TNE2Iteration(output, integration_container,
                      geometry_container, solver_container,
                      numerics_container, config_container,
                      surface_movement, grid_movement, FFDBox);
        break;
        
      case FLUID_STRUCTURE_EULER: case FLUID_STRUCTURE_NAVIER_STOKES:
        FluidStructureIteration(output, integration_container, geometry_container,
                                solver_container, numerics_container, config_container,
                                surface_movement, grid_movement, FFDBox);
        break;
        
      case WAVE_EQUATION:
        WaveIteration(output, integration_container, geometry_container,
                      solver_container, numerics_container, config_container,
                      surface_movement, grid_movement, FFDBox);
        break;
        
      case HEAT_EQUATION:
        HeatIteration(output, integration_container, geometry_container,
                      solver_container, numerics_container, config_container,
                      surface_movement, grid_movement, FFDBox);
        break;
        
      case POISSON_EQUATION:
        PoissonIteration(output, integration_container, geometry_container,
                         solver_container, numerics_container, config_container,
                         surface_movement, grid_movement, FFDBox);
        break;
        
      case LINEAR_ELASTICITY:
        FEAIteration(output, integration_container, geometry_container,
                     solver_container, numerics_container, config_container,
                     surface_movement, grid_movement, FFDBox);
        break;
        
      case ADJ_EULER: case ADJ_NAVIER_STOKES: case ADJ_RANS:
        AdjMeanFlowIteration(output, integration_container, geometry_container,
                             solver_container, numerics_container, config_container,
                             surface_movement, grid_movement, FFDBox);
        break;
        
      case ADJ_TNE2_EULER: case ADJ_TNE2_NAVIER_STOKES:
        AdjTNE2Iteration(output, integration_container, geometry_container,
                         solver_container, numerics_container, config_container,
                         surface_movement, grid_movement, FFDBox);
        break;
    }
    
    
    /*--- Synchronization point after a single solver iteration. Compute the
     wall clock time required. ---*/
    
#ifndef HAVE_MPI
    StopTime = double(clock())/double(CLOCKS_PER_SEC);
#else
    StopTime = MPI_Wtime();
#endif
    
    UsedTime = (StopTime - StartTime);
    
    /*--- For specific applications, evaluate and plot the equivalent area. ---*/
    
    if (config_container[ZONE_0]->GetEquivArea() == YES) {
      output->SetEquivalentArea(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
    }
    
    /*--- Check if there is any change in the runtime parameters ---*/
    
    CConfig *runtime = NULL;
    strcpy(runtime_file_name, "runtime.dat");
    runtime = new CConfig(runtime_file_name, config_container[ZONE_0]);
    
    /*--- Update the convergence history file (serial and parallel computations). ---*/
    
    output->SetConvHistory_Body(&ConvHist_file, geometry_container, solver_container,
                                config_container, integration_container, false, UsedTime, ZONE_0);
    
    /*--- Evaluate the new CFL number (adaptive). ---*/
    
    if (config_container[ZONE_0]->GetCFL_Adapt() == YES) {
      output->SetCFL_Number(solver_container, config_container, ZONE_0);
    }
    
    /*--- Check whether the current simulation has reached the specified
     convergence criteria, and set StopCalc to true, if so. ---*/
    
    switch (config_container[ZONE_0]->GetKind_Solver()) {
      case EULER: case NAVIER_STOKES: case RANS:
        StopCalc = integration_container[ZONE_0][FLOW_SOL]->GetConvergence(); break;
      case TNE2_EULER: case TNE2_NAVIER_STOKES:
        StopCalc = integration_container[ZONE_0][TNE2_SOL]->GetConvergence(); break;
      case WAVE_EQUATION:
        StopCalc = integration_container[ZONE_0][WAVE_SOL]->GetConvergence(); break;
      case HEAT_EQUATION:
        StopCalc = integration_container[ZONE_0][HEAT_SOL]->GetConvergence(); break;
      case LINEAR_ELASTICITY:
        StopCalc = integration_container[ZONE_0][FEA_SOL]->GetConvergence(); break;
      case ADJ_EULER: case ADJ_NAVIER_STOKES: case ADJ_RANS:
        StopCalc = integration_container[ZONE_0][ADJFLOW_SOL]->GetConvergence(); break;
      case ADJ_TNE2_EULER: case ADJ_TNE2_NAVIER_STOKES:
        StopCalc = integration_container[ZONE_0][ADJTNE2_SOL]->GetConvergence(); break;
    }
    
    /*--- Solution output. Determine whether a solution needs to be written
     after the current iteration, and if so, execute the output file writing
     routines. ---*/
    
    if ((ExtIter+1 >= config_container[ZONE_0]->GetnExtIter()) ||
        ((ExtIter % config_container[ZONE_0]->GetWrt_Sol_Freq() == 0) && (ExtIter != 0) &&
         !((config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_1ST) ||
           (config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_2ND))) ||
        (StopCalc) ||
        (((config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_1ST) ||
          (config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_2ND)) &&
         ((ExtIter == 0) || (ExtIter % config_container[ZONE_0]->GetWrt_Sol_Freq_DualTime() == 0)))) {
          
          /*--- Low-fidelity simulations (using a coarser multigrid level
           approximation to the solution) require an interpolation back to the
           finest grid. ---*/
          
          if (config_container[ZONE_0]->GetLowFidelitySim()) {
            integration_container[ZONE_0][FLOW_SOL]->SetProlongated_Solution(RUNTIME_FLOW_SYS, solver_container[ZONE_0][MESH_0][FLOW_SOL], solver_container[ZONE_0][MESH_1][FLOW_SOL], geometry_container[ZONE_0][MESH_0], geometry_container[ZONE_0][MESH_1], config_container[ZONE_0]);
            integration_container[ZONE_0][FLOW_SOL]->Smooth_Solution(RUNTIME_FLOW_SYS, solver_container[ZONE_0][MESH_0][FLOW_SOL], geometry_container[ZONE_0][MESH_0], 3, 1.25, config_container[ZONE_0]);
            solver_container[ZONE_0][MESH_0][config_container[ZONE_0]->GetContainerPosition(RUNTIME_FLOW_SYS)]->Set_MPI_Solution(geometry_container[ZONE_0][MESH_0], config_container[ZONE_0]);
            solver_container[ZONE_0][MESH_0][config_container[ZONE_0]->GetContainerPosition(RUNTIME_FLOW_SYS)]->Preprocessing(geometry_container[ZONE_0][MESH_0], solver_container[ZONE_0][MESH_0], config_container[ZONE_0], MESH_0, 0, RUNTIME_FLOW_SYS, false);
          }
          
          if (rank == MASTER_NODE) cout << endl << "-------------------------- File Output Summary --------------------------";
          
          /*--- Execute the routine for writing restart, volume solution,
           surface solution, and surface comma-separated value files. ---*/
          
          output->SetResult_Files(solver_container, geometry_container, config_container, ExtIter, nZone);
          
          /*--- Output a file with the forces breakdown. ---*/
          
          output->SetForces_Breakdown(geometry_container, solver_container,
                                      config_container, integration_container, ZONE_0);
          
          /*--- Compute the forces at different sections. ---*/
          
          if (config_container[ZONE_0]->GetPlot_Section_Forces()) {
            output->SetForceSections(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                     geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
          }
          
          if (rank == MASTER_NODE) cout << "-------------------------------------------------------------------------" << endl << endl;
          
        }
    
    /*--- If the convergence criteria has been met, terminate the simulation. ---*/
    
    if (StopCalc) break;
    
    ExtIter++;
    
  }
  
  /*--- Output some information to the console. ---*/
  
  if (rank == MASTER_NODE) {
    
    /*--- Print out the number of non-physical points and reconstructions ---*/
    
    if (config_container[ZONE_0]->GetNonphysical_Points() > 0)
      cout << "Warning: there are " << config_container[ZONE_0]->GetNonphysical_Points() << " non-physical points in the solution." << endl;
    if (config_container[ZONE_0]->GetNonphysical_Reconstr() > 0)
      cout << "Warning: " << config_container[ZONE_0]->GetNonphysical_Reconstr() << " reconstructed states for upwinding are non-physical." << endl;
    
    /*--- Close the convergence history file. ---*/
    
    ConvHist_file.close();
    cout << "History file, closed." << endl;
  }
  
  //  /*--- Deallocate config container ---*/
  //
  //  for (iZone = 0; iZone < nZone; iZone++) {
  //    if (config_container[iZone] != NULL) {
  //      delete config_container[iZone];
  //    }
  //  }
  //  if (config_container != NULL) delete[] config_container;
  
  
  /*--- Synchronization point after a single solver iteration. Compute the
   wall clock time required. ---*/
  
#ifndef HAVE_MPI
  StopTime = double(clock())/double(CLOCKS_PER_SEC);
#else
  StopTime = MPI_Wtime();
#endif
  
  /*--- Compute/print the total time for performance benchmarking. ---*/
  
  UsedTime = StopTime-StartTime;
  if (rank == MASTER_NODE) {
    cout << "\nCompleted in " << fixed << UsedTime << " seconds on "<< size;
    if (size == 1) cout << " core." << endl; else cout << " cores." << endl;
  }
  
  /*--- Exit the solver cleanly ---*/
  
  if (rank == MASTER_NODE)
    cout << endl <<"------------------------- Exit Success (SU2_CFD) ------------------------" << endl << endl;
  
#ifdef HAVE_MPI
  /*--- Finalize MPI parallelization ---*/
  MPI_Buffer_detach(&bptr, &bl);
  MPI_Finalize();
#endif
  
  return EXIT_SUCCESS;
  
}
Example #2
0
int main(int argc, char *argv[]) {
  
  bool StopCalc = false;
  su2double StartTime = 0.0, StopTime = 0.0, UsedTime = 0.0;
  unsigned long ExtIter = 0;
  unsigned short iMesh, iZone, nZone, nDim;
  char config_file_name[MAX_STRING_SIZE];
  char runtime_file_name[MAX_STRING_SIZE];
  ofstream ConvHist_file;
  int rank = MASTER_NODE;
  int size = SINGLE_NODE;
  
  /*--- MPI initialization, and buffer setting ---*/
  
#ifdef HAVE_MPI
  int *bptr, bl;
  SU2_MPI::Init(&argc, &argv);
  MPI_Buffer_attach( malloc(BUFSIZE), BUFSIZE );
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  MPI_Comm_size(MPI_COMM_WORLD, &size);
#endif
  
  /*--- Create pointers to all of the classes that may be used throughout
   the SU2_CFD code. In general, the pointers are instantiated down a
   heirarchy over all zones, multigrid levels, equation sets, and equation
   terms as described in the comments below. ---*/
  
  CDriver *driver                         = NULL;
  CIteration **iteration_container        = NULL;
  COutput *output                         = NULL;
  CIntegration ***integration_container   = NULL;
  CGeometry ***geometry_container         = NULL;
  CSolver ****solver_container            = NULL;
  CNumerics *****numerics_container       = NULL;
  CConfig **config_container              = NULL;
  CSurfaceMovement **surface_movement     = NULL;
  CVolumetricMovement **grid_movement     = NULL;
  CFreeFormDefBox*** FFDBox               = NULL;
  CInterpolator ***interpolator_container = NULL;
  CTransfer ***transfer_container         = NULL;
  
  /*--- Load in the number of zones and spatial dimensions in the mesh file (If no config
   file is specified, default.cfg is used) ---*/
  
  if (argc == 2) { strcpy(config_file_name, argv[1]); }
  else { strcpy(config_file_name, "default.cfg"); }
  
  /*--- Read the name and format of the input mesh file to get from the mesh
   file the number of zones and dimensions from the numerical grid (required
   for variables allocation)  ---*/
  
  CConfig *config = NULL;
  config = new CConfig(config_file_name, SU2_CFD);
  
  nZone = GetnZone(config->GetMesh_FileName(), config->GetMesh_FileFormat(), config);
  nDim  = GetnDim(config->GetMesh_FileName(), config->GetMesh_FileFormat());
  
  /*--- Definition and of the containers for all possible zones. ---*/
  
  iteration_container    = new CIteration*[nZone];
  solver_container       = new CSolver***[nZone];
  integration_container  = new CIntegration**[nZone];
  numerics_container     = new CNumerics****[nZone];
  config_container       = new CConfig*[nZone];
  geometry_container     = new CGeometry**[nZone];
  surface_movement       = new CSurfaceMovement*[nZone];
  grid_movement          = new CVolumetricMovement*[nZone];
  FFDBox                 = new CFreeFormDefBox**[nZone];
  interpolator_container = new CInterpolator**[nZone];
  transfer_container     = new CTransfer**[nZone];
  
  for (iZone = 0; iZone < nZone; iZone++) {
    solver_container[iZone]       = NULL;
    integration_container[iZone]  = NULL;
    numerics_container[iZone]     = NULL;
    config_container[iZone]       = NULL;
    geometry_container[iZone]     = NULL;
    surface_movement[iZone]       = NULL;
    grid_movement[iZone]          = NULL;
    FFDBox[iZone]                 = NULL;
    interpolator_container[iZone] = NULL;
    transfer_container[iZone]     = NULL;
  }
  
  /*--- Loop over all zones to initialize the various classes. In most
   cases, nZone is equal to one. This represents the solution of a partial
   differential equation on a single block, unstructured mesh. ---*/
  
  for (iZone = 0; iZone < nZone; iZone++) {
    
    /*--- Definition of the configuration option class for all zones. In this
     constructor, the input configuration file is parsed and all options are
     read and stored. ---*/
    
    config_container[iZone] = new CConfig(config_file_name, SU2_CFD, iZone, nZone, nDim, VERB_HIGH);
    
    /*--- Definition of the geometry class to store the primal grid in the
     partitioning process. ---*/
    
    CGeometry *geometry_aux = NULL;
    
    /*--- All ranks process the grid and call ParMETIS for partitioning ---*/
    
    geometry_aux = new CPhysicalGeometry(config_container[iZone], iZone, nZone);
    
    /*--- Color the initial grid and set the send-receive domains (ParMETIS) ---*/
    
    geometry_aux->SetColorGrid_Parallel(config_container[iZone]);
    
    /*--- Allocate the memory of the current domain, and divide the grid
     between the ranks. ---*/
    
    geometry_container[iZone] = new CGeometry *[config_container[iZone]->GetnMGLevels()+1];
    geometry_container[iZone][MESH_0] = new CPhysicalGeometry(geometry_aux, config_container[iZone], 1);
    
    /*--- Deallocate the memory of geometry_aux ---*/
    
    delete geometry_aux;
    
    /*--- Add the Send/Receive boundaries ---*/
    
    geometry_container[iZone][MESH_0]->SetSendReceive(config_container[iZone]);
    
    /*--- Add the Send/Receive boundaries ---*/
    
    geometry_container[iZone][MESH_0]->SetBoundaries(config_container[iZone]);
    
  }

  if (rank == MASTER_NODE)
    cout << endl <<"------------------------- Geometry Preprocessing ------------------------" << endl;
  
  /*--- Preprocessing of the geometry for all zones. In this routine, the edge-
   based data structure is constructed, i.e. node and cell neighbors are
   identified and linked, face areas and volumes of the dual mesh cells are
   computed, and the multigrid levels are created using an agglomeration procedure. ---*/
  
  Geometrical_Preprocessing(geometry_container, config_container, nZone);
  
  for (iZone = 0; iZone < nZone; iZone++) {
    
    /*--- Computation of wall distances for turbulence modeling ---*/
    
    if (rank == MASTER_NODE)
      cout << "Computing wall distances." << endl;

    if ((config_container[iZone]->GetKind_Solver() == RANS) ||
        (config_container[iZone]->GetKind_Solver() == ADJ_RANS) ||
        (config_container[iZone]->GetKind_Solver() == DISC_ADJ_RANS))
      geometry_container[iZone][MESH_0]->ComputeWall_Distance(config_container[iZone]);
    
    /*--- Computation of positive surface area in the z-plane which is used for
     the calculation of force coefficient (non-dimensionalization). ---*/
    
    geometry_container[iZone][MESH_0]->SetPositive_ZArea(config_container[iZone]);
    
    /*--- Set the near-field, interface and actuator disk boundary conditions, if necessary. ---*/
    
    for (iMesh = 0; iMesh <= config_container[iZone]->GetnMGLevels(); iMesh++) {
      geometry_container[iZone][iMesh]->MatchNearField(config_container[iZone]);
      geometry_container[iZone][iMesh]->MatchInterface(config_container[iZone]);
      geometry_container[iZone][iMesh]->MatchActuator_Disk(config_container[iZone]);
    }
    
  }
  
  /*--- If activated by the compile directive, perform a partition analysis. ---*/
#if PARTITION
  Partition_Analysis(geometry_container[ZONE_0][MESH_0], config_container[ZONE_0]);
#endif
  
  if (rank == MASTER_NODE)
    cout << endl <<"------------------------- Driver Preprocessing --------------------------" << endl;
  
  /*--- First, given the basic information about the number of zones and the
   solver types from the config, instantiate the appropriate driver for the problem. ---*/
  
  Driver_Preprocessing(&driver, iteration_container, solver_container,
                       geometry_container, integration_container, numerics_container,
                       interpolator_container, transfer_container, config_container, nZone, nDim);
  
  
  /*--- Instantiate the geometry movement classes for the solution of unsteady
   flows on dynamic meshes, including rigid mesh transformations, dynamically
   deforming meshes, and time-spectral preprocessing. ---*/
  
  for (iZone = 0; iZone < nZone; iZone++) {
    
    if (config_container[iZone]->GetGrid_Movement() ||
        (config_container[iZone]->GetDirectDiff() == D_DESIGN)) {
      if (rank == MASTER_NODE)
        cout << "Setting dynamic mesh structure." << endl;
      grid_movement[iZone] = new CVolumetricMovement(geometry_container[iZone][MESH_0], config_container[iZone]);
      FFDBox[iZone] = new CFreeFormDefBox*[MAX_NUMBER_FFD];
      surface_movement[iZone] = new CSurfaceMovement();
      surface_movement[iZone]->CopyBoundary(geometry_container[iZone][MESH_0], config_container[iZone]);
      if (config_container[iZone]->GetUnsteady_Simulation() == TIME_SPECTRAL)
        SetGrid_Movement(geometry_container[iZone], surface_movement[iZone], grid_movement[iZone],
                         FFDBox[iZone], solver_container[iZone], config_container[iZone], iZone, 0, 0);
    }
    
    if (config_container[iZone]->GetDirectDiff() == D_DESIGN){
      if (rank == MASTER_NODE)
        cout << "Setting surface/volume derivatives." << endl;
      
      /*--- Set the surface derivatives, i.e. the derivative of the surface mesh nodes with respect to the design variables ---*/
      
      surface_movement[iZone]->SetSurface_Derivative(geometry_container[iZone][MESH_0],config_container[iZone]);
      
      /*--- Call the volume deformation routine with derivative mode enabled.
       This computes the derivative of the volume mesh with respect to the surface nodes ---*/
      
      grid_movement[iZone]->SetVolume_Deformation(geometry_container[iZone][MESH_0],config_container[iZone], true, true);
      
      /*--- Update the multi-grid structure to propagate the derivative information to the coarser levels ---*/
      
      geometry_container[iZone][MESH_0]->UpdateGeometry(geometry_container[iZone],config_container[iZone]);
      
      /*--- Set the derivative of the wall-distance with respect to the surface nodes ---*/
      
      if ( (config_container[iZone]->GetKind_Solver() == RANS) ||
          (config_container[iZone]->GetKind_Solver() == ADJ_RANS) ||
          (config_container[iZone]->GetKind_Solver() == DISC_ADJ_RANS))
        geometry_container[iZone][MESH_0]->ComputeWall_Distance(config_container[iZone]);
    }
    
    
  }
  
  /*--- Coupling between zones (limited to two zones at the moment) ---*/

  bool fsi = config_container[ZONE_0]->GetFSI_Simulation();
  
  if ((nZone == 2) && !(fsi)) {
    if (rank == MASTER_NODE)
      cout << endl <<"--------------------- Setting Coupling Between Zones --------------------" << endl;
    geometry_container[ZONE_0][MESH_0]->MatchZone(config_container[ZONE_0], geometry_container[ZONE_1][MESH_0],
                                                  config_container[ZONE_1], ZONE_0, nZone);
    geometry_container[ZONE_1][MESH_0]->MatchZone(config_container[ZONE_1], geometry_container[ZONE_0][MESH_0],
                                                  config_container[ZONE_0], ZONE_1, nZone);
  }
  
  /*--- Definition of the output class (one for all zones). The output class
   manages the writing of all restart, volume solution, surface solution,
   surface comma-separated value, and convergence history files (both in serial
   and in parallel). ---*/
  
  output = new COutput();
  
  /*--- Open the convergence history file ---*/
  
  if (rank == MASTER_NODE)
    output->SetConvHistory_Header(&ConvHist_file, config_container[ZONE_0]);
  
  /*--- Check for an unsteady restart. Update ExtIter if necessary. ---*/
  if (config_container[ZONE_0]->GetWrt_Unsteady() && config_container[ZONE_0]->GetRestart())
    ExtIter = config_container[ZONE_0]->GetUnst_RestartIter();
  
  /*--- Check for a dynamic restart (structural analysis). Update ExtIter if necessary. ---*/
  if (config_container[ZONE_0]->GetKind_Solver() == FEM_ELASTICITY
		  && config_container[ZONE_0]->GetWrt_Dynamic() && config_container[ZONE_0]->GetRestart())
	  	  ExtIter = config_container[ZONE_0]->GetDyn_RestartIter();

  /*--- Initiate value at each interface for the mixing plane ---*/
  if(config_container[ZONE_0]->GetBoolMixingPlane())
  	for (iZone = 0; iZone < nZone; iZone++)
  	  iteration_container[iZone]->Preprocess(output, integration_container, geometry_container, solver_container, numerics_container, config_container, surface_movement, grid_movement, FFDBox, iZone);

  /*--- Main external loop of the solver. Within this loop, each iteration ---*/
  
  if (rank == MASTER_NODE)
    cout << endl <<"------------------------------ Begin Solver -----------------------------" << endl;
  
  /*--- Set up a timer for performance benchmarking (preprocessing time is not included) ---*/
  
#ifndef HAVE_MPI
  StartTime = su2double(clock())/su2double(CLOCKS_PER_SEC);
#else
  StartTime = MPI_Wtime();
#endif

  /*--- This is temporal and just to check. It will have to be added to the regular history file ---*/
  
  ofstream historyFile_FSI;
  bool writeHistFSI = config_container[ZONE_0]->GetWrite_Conv_FSI();
  if (writeHistFSI && (rank == MASTER_NODE)){
    char cstrFSI[200];
    string filenameHistFSI = config_container[ZONE_0]->GetConv_FileName_FSI();
    strcpy (cstrFSI, filenameHistFSI.data());
    historyFile_FSI.open (cstrFSI);
    historyFile_FSI << "Time,Iteration,Aitken,URes,logResidual,orderMagnResidual" << endl;
    historyFile_FSI.close();
  }

  while (ExtIter < config_container[ZONE_0]->GetnExtIter()) {
    
    /*--- Set the value of the external iteration. ---*/
	for (iZone = 0; iZone < nZone; iZone++) config_container[iZone]->SetExtIter(ExtIter);

    
    /*--- Read the target pressure ---*/
    
    if (config_container[ZONE_0]->GetInvDesign_Cp() == YES)
      output->SetCp_InverseDesign(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                  geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
    
    /*--- Read the target heat flux ---*/
    
    if (config_container[ZONE_0]->GetInvDesign_HeatFlux() == YES)
      output->SetHeat_InverseDesign(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                    geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
    
    /*--- Perform a single iteration of the chosen PDE solver. ---*/

      /*--- Run a single iteration of the problem using the driver class. ---*/

      driver->Run(iteration_container, output, integration_container,
                  geometry_container, solver_container, numerics_container,
                  config_container, surface_movement, grid_movement, FFDBox,
                  interpolator_container, transfer_container);
      
    
    /*--- Synchronization point after a single solver iteration. Compute the
     wall clock time required. ---*/
    
#ifndef HAVE_MPI
    StopTime = su2double(clock())/su2double(CLOCKS_PER_SEC);
#else
    StopTime = MPI_Wtime();
#endif
    
    UsedTime = (StopTime - StartTime);
    
    /*--- For specific applications, evaluate and plot the equivalent area. ---*/
    
    if (config_container[ZONE_0]->GetEquivArea() == YES) {
      output->SetEquivalentArea(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
    }
    
    /*--- Check if there is any change in the runtime parameters ---*/
    
    CConfig *runtime = NULL;
    strcpy(runtime_file_name, "runtime.dat");
    runtime = new CConfig(runtime_file_name, config_container[ZONE_0]);
    runtime->SetExtIter(ExtIter);
    
	/*--- Update the convergence history file (serial and parallel computations). ---*/

	if (!fsi){
		output->SetConvHistory_Body(&ConvHist_file, geometry_container, solver_container,
				config_container, integration_container, false, UsedTime, ZONE_0);

	}

    
    /*--- Evaluate the new CFL number (adaptive). ---*/
    
    if (config_container[ZONE_0]->GetCFL_Adapt() == YES) {
      output->SetCFL_Number(solver_container, config_container, ZONE_0);
    }
    
    /*--- Check whether the current simulation has reached the specified
     convergence criteria, and set StopCalc to true, if so. ---*/
    
    switch (config_container[ZONE_0]->GetKind_Solver()) {
      case EULER: case NAVIER_STOKES: case RANS:
        StopCalc = integration_container[ZONE_0][FLOW_SOL]->GetConvergence(); break;
      case WAVE_EQUATION:
        StopCalc = integration_container[ZONE_0][WAVE_SOL]->GetConvergence(); break;
      case HEAT_EQUATION:
        StopCalc = integration_container[ZONE_0][HEAT_SOL]->GetConvergence(); break;
	  case FEM_ELASTICITY:
	    StopCalc = integration_container[ZONE_0][FEA_SOL]->GetConvergence(); break;
      case ADJ_EULER: case ADJ_NAVIER_STOKES: case ADJ_RANS:
      case DISC_ADJ_EULER: case DISC_ADJ_NAVIER_STOKES: case DISC_ADJ_RANS:
        StopCalc = integration_container[ZONE_0][ADJFLOW_SOL]->GetConvergence(); break;
    }
    
		/*--- Solution output. Determine whether a solution needs to be written
		 after the current iteration, and if so, execute the output file writing
		 routines. ---*/
		
		if ((ExtIter+1 >= config_container[ZONE_0]->GetnExtIter())
				
				||
				
				((ExtIter % config_container[ZONE_0]->GetWrt_Sol_Freq() == 0) && (ExtIter != 0) &&
				 !((config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_1ST) ||
					 (config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_2ND) ||
					 (config_container[ZONE_0]->GetUnsteady_Simulation() == TIME_STEPPING)))
				
				||
				
				(StopCalc)
				
				||
				
				(((config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_1ST) ||
				 (config_container[ZONE_0]->GetUnsteady_Simulation() == TIME_STEPPING)) &&
				 ((ExtIter == 0) || (ExtIter % config_container[ZONE_0]->GetWrt_Sol_Freq_DualTime() == 0)))
				
				||
				
				((config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_2ND) && (!fsi) &&
				 ((ExtIter == 0) || ((ExtIter % config_container[ZONE_0]->GetWrt_Sol_Freq_DualTime() == 0) ||
														 ((ExtIter-1) % config_container[ZONE_0]->GetWrt_Sol_Freq_DualTime() == 0))))
				
				||
				
				((config_container[ZONE_0]->GetUnsteady_Simulation() == DT_STEPPING_2ND) && (fsi) &&
				 ((ExtIter == 0) || ((ExtIter % config_container[ZONE_0]->GetWrt_Sol_Freq_DualTime() == 0))))

				||

				(((config_container[ZONE_0]->GetDynamic_Analysis() == DYNAMIC) &&
						 ((ExtIter == 0) || (ExtIter % config_container[ZONE_0]->GetWrt_Sol_Freq_DualTime() == 0))))){

					
          /*--- Low-fidelity simulations (using a coarser multigrid level
           approximation to the solution) require an interpolation back to the
           finest grid. ---*/
          
          if (config_container[ZONE_0]->GetLowFidelitySim()) {
            integration_container[ZONE_0][FLOW_SOL]->SetProlongated_Solution(RUNTIME_FLOW_SYS, solver_container[ZONE_0][MESH_0][FLOW_SOL], solver_container[ZONE_0][MESH_1][FLOW_SOL], geometry_container[ZONE_0][MESH_0], geometry_container[ZONE_0][MESH_1], config_container[ZONE_0]);
            integration_container[ZONE_0][FLOW_SOL]->Smooth_Solution(RUNTIME_FLOW_SYS, solver_container[ZONE_0][MESH_0][FLOW_SOL], geometry_container[ZONE_0][MESH_0], 3, 1.25, config_container[ZONE_0]);
            solver_container[ZONE_0][MESH_0][config_container[ZONE_0]->GetContainerPosition(RUNTIME_FLOW_SYS)]->Set_MPI_Solution(geometry_container[ZONE_0][MESH_0], config_container[ZONE_0]);
            solver_container[ZONE_0][MESH_0][config_container[ZONE_0]->GetContainerPosition(RUNTIME_FLOW_SYS)]->Preprocessing(geometry_container[ZONE_0][MESH_0], solver_container[ZONE_0][MESH_0], config_container[ZONE_0], MESH_0, 0, RUNTIME_FLOW_SYS, true);
          }


          if (rank == MASTER_NODE) cout << endl << "-------------------------- File Output Summary --------------------------";
          
          /*--- Execute the routine for writing restart, volume solution,
           surface solution, and surface comma-separated value files. ---*/
          
          output->SetResult_Files(solver_container, geometry_container, config_container, ExtIter, nZone);
          
          /*--- Output a file with the forces breakdown. ---*/
          
          output->SetForces_Breakdown(geometry_container, solver_container,
                                      config_container, integration_container, ZONE_0);
          
          /*--- Compute the forces at different sections. ---*/
          
          if (config_container[ZONE_0]->GetPlot_Section_Forces()) {
            output->SetForceSections(solver_container[ZONE_0][MESH_0][FLOW_SOL],
                                     geometry_container[ZONE_0][MESH_0], config_container[ZONE_0], ExtIter);
          }
          
          if (rank == MASTER_NODE) cout << "-------------------------------------------------------------------------" << endl << endl;
          
        }
    
    /*--- If the convergence criteria has been met, terminate the simulation. ---*/
    
    if (StopCalc) break;
    
    ExtIter++;
    
  }
  
  /*--- Output some information to the console. ---*/
  
  if (rank == MASTER_NODE) {
    
    /*--- Print out the number of non-physical points and reconstructions ---*/
    
    if (config_container[ZONE_0]->GetNonphysical_Points() > 0)
      cout << "Warning: there are " << config_container[ZONE_0]->GetNonphysical_Points() << " non-physical points in the solution." << endl;
    if (config_container[ZONE_0]->GetNonphysical_Reconstr() > 0)
      cout << "Warning: " << config_container[ZONE_0]->GetNonphysical_Reconstr() << " reconstructed states for upwinding are non-physical." << endl;
    
    /*--- Close the convergence history file. ---*/
    
    ConvHist_file.close();
    cout << "History file, closed." << endl;
  }
  
  //  /*--- Deallocate config container ---*/
  //
  //  for (iZone = 0; iZone < nZone; iZone++) {
  //    if (config_container[iZone] != NULL) {
  //      delete config_container[iZone];
  //    }
  //  }
  //  if (config_container != NULL) delete[] config_container;
  
  
  /*--- Synchronization point after a single solver iteration. Compute the
   wall clock time required. ---*/
  
#ifndef HAVE_MPI
  StopTime = su2double(clock())/su2double(CLOCKS_PER_SEC);
#else
  StopTime = MPI_Wtime();
#endif
  
  /*--- Compute/print the total time for performance benchmarking. ---*/
  
  UsedTime = StopTime-StartTime;
  if (rank == MASTER_NODE) {
    cout << "\nCompleted in " << fixed << UsedTime << " seconds on "<< size;
    if (size == 1) cout << " core." << endl; else cout << " cores." << endl;
  }
  
  /*--- Exit the solver cleanly ---*/
  
  if (rank == MASTER_NODE)
    cout << endl <<"------------------------- Exit Success (SU2_CFD) ------------------------" << endl << endl;
  
#ifdef HAVE_MPI
  /*--- Finalize MPI parallelization ---*/
  MPI_Buffer_detach(&bptr, &bl);
  MPI_Finalize();
#endif
  
  return EXIT_SUCCESS;
  
}