// ---------------------------------------------------------------
// advance solution by one timestep
// return max safe timestep
Real
AMRNavierStokes::advance()
{
if (s_verbosity >= 2)
{
pout () << "AMRNavierStokes::advance " << m_level
<< ", starting time = "
<< setiosflags(ios::fixed) << setprecision(6)
<< setw(12) << m_time
<< ", dt = "
<< setiosflags(ios::fixed) << setprecision(6) << setw(12) << dt()
<< endl;
}
#ifdef DEBUG
// if we're at the beginning of a composite timestep,
// save current multilevel state
if (m_level == 0)
{
AMRNavierStokes* levelPtr = this;
// figure out finest level
while (!levelPtr->finestLevel())
{
levelPtr = levelPtr->finerNSPtr();
}
int finest_level = levelPtr->m_level;
levelPtr = this;
for (int lev=0; lev<= finest_level; lev++)
{
levelPtr->m_saved_time = m_time;
levelPtr->newVel().copyTo(levelPtr->newVel().interval(),
*levelPtr->m_vel_save_ptr,
levelPtr->m_vel_save_ptr->interval());
levelPtr->newLambda().copyTo(levelPtr->newLambda().interval(),
*levelPtr->m_lambda_save_ptr,
levelPtr->m_lambda_save_ptr->interval());
for (int comp=0; comp<s_num_scal_comps; comp++)
{
levelPtr->newScal(comp).copyTo(levelPtr->newScal(comp).interval(),
*levelPtr->m_scal_save[comp],
levelPtr->m_scal_save[comp]->interval());
}
levelPtr = levelPtr->finerNSPtr();
}
}
#endif
Real old_time = m_time;
Real new_time = m_time + m_dt;
m_dt_save = m_dt;
swapOldAndNewStates();
const DisjointBoxLayout levelGrids = newVel().getBoxes();
// initialize flux registers
if (!finestLevel())
{
m_flux_register.setToZero();
for (int comp=0; comp<s_num_scal_comps; comp++)
{
m_scal_fluxreg_ptrs[comp]->setToZero();
}
m_lambda_flux_reg.setToZero();
}
// compute advection velocities -- if we're using the
// patchGodunov approach, these will have ghost cells.
IntVect advVelGhost(IntVect::Unit);
LevelData<FluxBox> advectionVel(levelGrids, 1, advVelGhost);
if (s_set_bogus_values)
{
setValLevel(advectionVel, s_bogus_value);
}
computeAdvectionVelocities(advectionVel);
// now that advection velocities have been computed,
// update advected scalars
// do scalar boundary conditions on lambda
// (also grow to appropriate size)
LevelData<FArrayBox> lambdaTemp;
fillLambda(lambdaTemp, old_time);
LevelFluxRegister* crseLambdaFluxRegPtr = NULL;
if (m_level > 0)
{
crseLambdaFluxRegPtr = &(crseNSPtr()->m_lambda_flux_reg);
}
advectScalar(newLambda(), lambdaTemp, advectionVel,
crseLambdaFluxRegPtr, m_lambda_flux_reg,
m_patchGodLambda,
m_dt);
// now do advected-diffused scalars
if (s_num_scal_comps > 0)
{
// loop over scalar components
for (int comp=0; comp<s_num_scal_comps; comp++)
{
BCHolder scalPhysBCs = m_physBCPtr->scalarTraceFuncBC(comp);
LevelData<FArrayBox> scalarTemp;
fillScalars(scalarTemp, old_time,comp);
// now get coarse-level scalars for BC's if necessary
LevelData<FArrayBox>* newCrseScalPtr = NULL;
LevelData<FArrayBox>* oldCrseScalPtr = NULL;
Real oldCrseTime = -s_bogus_value;
Real newCrseTime = s_bogus_value;
LevelFluxRegister* crseScalFluxRegPtr = NULL;
if (m_level > 0)
{
newCrseScalPtr = &(crseNSPtr()->newScal(comp));
oldCrseScalPtr = &(crseNSPtr()->oldScal(comp));
newCrseTime = crseNSPtr()->time();
oldCrseTime = newCrseTime - crseNSPtr()->dt();
crseScalFluxRegPtr = (crseNSPtr()->m_scal_fluxreg_ptrs[comp]);
}
advectDiffuseScalar(newScal(comp), scalarTemp,
advectionVel,
s_scal_coeffs[comp],
oldCrseScalPtr, newCrseScalPtr,
oldCrseTime, newCrseTime,
crseScalFluxRegPtr,
*(m_scal_fluxreg_ptrs[comp]),
*(m_patchGodScalars[comp]),
scalPhysBCs,
m_dt, comp);
} // end loop over components
} // end advected-diffused scalars
// now predict velocities -- this returns the advection term
// put this in "new_vel"
LevelData<FArrayBox>& uStar = newVel();
predictVelocities(uStar, advectionVel);
computeUStar(uStar);
// if a coarser level exists, will need coarse-level data for proj
LevelData<FArrayBox>* crseVelPtr = NULL;
if (m_level > 0)
{
const DisjointBoxLayout& crseGrids = crseNSPtr()->newVel().getBoxes();
crseVelPtr = new LevelData<FArrayBox>(crseGrids, SpaceDim);
// coarse velocity BC data is interpolated in time
crseNSPtr()->fillVelocity(*crseVelPtr, new_time);
}
// need to do physical boundary conditions and exchanges
bool isViscous = (s_nu > 0);
VelBCHolder velBC(m_physBCPtr->uStarFuncBC(isViscous));
velBC.applyBCs(uStar, levelGrids,
m_problem_domain, m_dx,
false); // inhomogeneous
// noel -- all time in level project
m_projection.LevelProject(uStar, crseVelPtr, new_time, m_dt);
// as things stand now, physical BC's are re-set in LevelProjection
// compute maximum safe timestep for next iteration
Real newDt = computeDt();
// clean up temp storage
if (crseVelPtr != NULL)
{
delete crseVelPtr;
crseVelPtr = NULL;
}
++m_level_steps;
if (s_verbosity >= 2)
{
pout () << "AMRNavierStokes::advance " << m_level
<< ", end time = "
<< setiosflags(ios::fixed) << setprecision(6) << setw(12) << m_time
<< ", dt = "
<< setiosflags(ios::fixed) << setprecision(6)
<< setw(12) << dt()
<< endl;
}
return newDt;
}
void timeStepper::timeStep(int &numReads, int &numWrites)
{
af::timer timeStepTimer = af::timer::start();
PetscPrintf(PETSC_COMM_WORLD, " Time = %f, dt = %f\n\n", time, dt);
int numReadsDt, numWritesDt;
computeDt(numReadsDt, numWritesDt);
/* First take a half step */
PetscPrintf(PETSC_COMM_WORLD, " ---Half step--- \n");
af::timer halfStepTimer = af::timer::start();
currentStep = timeStepperSwitches::HALF_STEP;
/* Apply boundary conditions on primOld */
af::timer boundaryTimer = af::timer::start();
boundaries::applyBoundaryConditions(boundaryLeft, boundaryRight,
boundaryTop, boundaryBottom,
boundaryFront, boundaryBack,
*primOld
);
setProblemSpecificBCs(numReads,numWrites);
double boundaryTime = af::timer::stop(boundaryTimer);
int numReadsElemSet, numWritesElemSet;
int numReadsComputeFluxes, numWritesComputeFluxes;
af::timer elemOldTimer = af::timer::start();
elemOld->set(*primOld, *geomCenter,
numReadsElemSet, numWritesElemSet
);
double elemOldTime = af::timer::stop(elemOldTimer);
af::timer consOldTimer = af::timer::start();
elemOld->computeFluxes(0, *consOld,
numReadsComputeFluxes, numWritesComputeFluxes
);
numReads = numReadsElemSet + numReadsComputeFluxes;
numWrites = numWritesElemSet + numWritesComputeFluxes;
double consOldTime = af::timer::stop(consOldTimer);
int numReadsExplicitSouces, numWritesExplicitSouces;
af::timer explicitSourcesTimer = af::timer::start();
double dX[3];
dX[0] = XCoords->dX1;
dX[1] = XCoords->dX2;
dX[2] = XCoords->dX3;
elemOld->computeExplicitSources(dX, *sourcesExplicit,
numReadsExplicitSouces,
numWritesExplicitSouces
);
numReads += numReadsExplicitSouces;
numWrites += numWritesExplicitSouces;
double explicitSourcesTime = af::timer::stop(explicitSourcesTimer);
int numReadsImplicitSources, numWritesImplicitSources;
elemOld->computeImplicitSources(*sourcesImplicitOld,
elemOld->tau,
numReadsImplicitSources,
numWritesImplicitSources
);
numReads += numReadsImplicitSources;
numWrites += numWritesImplicitSources;
af::timer divFluxTimer = af::timer::start();
int numReadsDivFluxes, numWritesDivFluxes;
computeDivOfFluxes(*primOld, numReadsDivFluxes, numWritesDivFluxes);
numReads += numReadsDivFluxes;
numWrites += numWritesDivFluxes;
double divFluxTime = af::timer::stop(divFluxTimer);
/* Set a guess for prim */
for (int var=0; var < vars::numFluidVars; var++)
{
prim->vars[var] = primOld->vars[var];
numReads += 1;
numWrites += 1;
}
af::timer inductionEqnTimer = af::timer::start();
cons->vars[vars::B1] =
consOld->vars[vars::B1] - 0.5*dt*divFluxes->vars[vars::B1];
cons->vars[vars::B2] =
consOld->vars[vars::B2] - 0.5*dt*divFluxes->vars[vars::B2];
cons->vars[vars::B3] =
consOld->vars[vars::B3] - 0.5*dt*divFluxes->vars[vars::B3];
prim->vars[vars::B1] = cons->vars[vars::B1]/geomCenter->g;
prim->vars[vars::B1].eval();
prim->vars[vars::B2] = cons->vars[vars::B2]/geomCenter->g;
prim->vars[vars::B2].eval();
prim->vars[vars::B3] = cons->vars[vars::B3]/geomCenter->g;
prim->vars[vars::B3].eval();
double inductionEqnTime = af::timer::stop(inductionEqnTimer);
primGuessPlusEps->vars[vars::B1] = prim->vars[vars::B1];
primGuessPlusEps->vars[vars::B2] = prim->vars[vars::B2];
primGuessPlusEps->vars[vars::B3] = prim->vars[vars::B3];
primGuessLineSearchTrial->vars[vars::B1] = prim->vars[vars::B1];
primGuessLineSearchTrial->vars[vars::B2] = prim->vars[vars::B2];
primGuessLineSearchTrial->vars[vars::B3] = prim->vars[vars::B3];
/* Solve dU/dt + div.F - S = 0 to get prim at n+1/2 */
jacobianAssemblyTime = 0.;
lineSearchTime = 0.;
linearSolverTime = 0.;
/* Use simple ideal solver if able and requested */
af::sync();
af::timer solverTimer;
af::timer stepConsTimer;
double solverTime, stepConsTime;
if (params::conduction == 0 &&
params::viscosity == 0 &&
params::solver == solvers::IDEAL)
{
stepConsTimer = af::timer::start();
timeStepFluidCons(0.5*dt);
af::sync();
stepConsTime = af::timer::stop(stepConsTimer);
int numreads, numwrites;
solverTimer = af::timer::start();
idealSolver(*prim, numreads, numwrites);
af::sync();
solverTime = af::timer::stop(solverTimer);
} else {
solverTimer = af::timer::start();
solve(*prim);
solverTime = af::timer::stop(solverTimer);
}
//double solverTime = af::timer::stop(solverTimer);
/* Copy solution to primHalfStepGhosted. WARNING: Right now
* primHalfStep->vars[var] points to prim->vars[var]. Might need to do a deep
* copy. */
for (int var=0; var < prim->numVars; var++)
{
primHalfStep->vars[var] = prim->vars[var];
numReads += 1;
numWrites += 1;
}
af::timer halfStepCommTimer = af::timer::start();
primHalfStep->communicate();
double halfStepCommTime = af::timer::stop(halfStepCommTimer);
af::timer halfStepDiagTimer = af::timer::start();
halfStepDiagnostics(numReads,numWrites);
double halfStepDiagTime = af::timer::stop(halfStepDiagTimer);
/* Half step complete */
double halfStepTime = af::timer::stop(halfStepTimer);
/*PetscPrintf(PETSC_COMM_WORLD, "\n");
PetscPrintf(PETSC_COMM_WORLD, " ---Performance report--- \n");
PetscPrintf(PETSC_COMM_WORLD, " Boundary Conditions : %g secs, %g %\n",
boundaryTime, boundaryTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Setting elemOld : %g secs, %g %\n",
elemOldTime,
elemOldTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Conserved vars Old : %g secs, %g %\n",
consOldTime, consOldTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Explicit sources : %g secs, %g %\n",
explicitSourcesTime,
explicitSourcesTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Divergence of fluxes: %g secs, %g %\n",
divFluxTime, divFluxTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Nonlinear solver : %g secs, %g %\n",
solverTime, solverTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " |- Jacobian assembly: %g secs, %g %\n",
jacobianAssemblyTime,
jacobianAssemblyTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " |- Linear solver : %g secs, %g %\n",
linearSolverTime,
linearSolverTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " |- Linesearch : %g secs, %g %\n",
lineSearchTime,
lineSearchTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Fluid cons : %g secs, %g %\n",
stepConsTime,
stepConsTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Induction equation : %g secs, %g %\n",
inductionEqnTime,
inductionEqnTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Communication : %g secs, %g %\n",
halfStepCommTime,
halfStepCommTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Diagnostics : %g secs, %g %\n",
halfStepDiagTime,
halfStepDiagTime/halfStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Half step time : %g secs\n\n",
halfStepTime
);*/
/* Now take the full step */
PetscPrintf(PETSC_COMM_WORLD, " ---Full step--- \n");
af::sync();
af::timer fullStepTimer = af::timer::start();
currentStep = timeStepperSwitches::FULL_STEP;
/* apply boundary conditions on primHalfStep */
boundaryTimer = af::timer::start();
boundaries::applyBoundaryConditions(boundaryLeft, boundaryRight,
boundaryTop, boundaryBottom,
boundaryFront, boundaryBack,
*primHalfStep
);
setProblemSpecificBCs(numReads,numWrites);
af::sync();
boundaryTime = af::timer::stop(boundaryTimer);
af::timer elemHalfStepTimer = af::timer::start();
elemHalfStep->set(*primHalfStep, *geomCenter,
numReadsElemSet, numWritesElemSet
);
numReads += numReadsElemSet;
numWrites += numWritesElemSet;
af::sync();
double elemHalfStepTime = af::timer::stop(elemHalfStepTimer);
explicitSourcesTimer = af::timer::start();
elemHalfStep->computeExplicitSources(dX, *sourcesExplicit,
numReadsExplicitSouces,
numWritesExplicitSouces
);
numReads += numReadsExplicitSouces;
numWrites += numWritesExplicitSouces;
af::sync();
explicitSourcesTime = af::timer::stop(explicitSourcesTimer);
af::timer implicitSourcesTimer = af::timer::start();
elemOld->computeImplicitSources(*sourcesImplicitOld,
elemHalfStep->tau,
numReadsImplicitSources,
numWritesImplicitSources
);
numReads += numReadsImplicitSources;
numWrites += numWritesImplicitSources;
af::sync();
double implicitSourcesTime = af::timer::stop(implicitSourcesTimer);
divFluxTimer = af::timer::start();
computeDivOfFluxes(*primHalfStep, numReadsDivFluxes, numWritesDivFluxes);
numReads += numReadsDivFluxes;
numWrites += numWritesDivFluxes;
af::sync();
divFluxTime = af::timer::stop(divFluxTimer);
inductionEqnTimer = af::timer::start();
cons->vars[vars::B1] =
consOld->vars[vars::B1] - dt*divFluxes->vars[vars::B1];
cons->vars[vars::B2] =
consOld->vars[vars::B2] - dt*divFluxes->vars[vars::B2];
cons->vars[vars::B3] =
consOld->vars[vars::B3] - dt*divFluxes->vars[vars::B3];
prim->vars[vars::B1] = cons->vars[vars::B1]/geomCenter->g;
prim->vars[vars::B1].eval();
prim->vars[vars::B2] = cons->vars[vars::B2]/geomCenter->g;
prim->vars[vars::B2].eval();
prim->vars[vars::B3] = cons->vars[vars::B3]/geomCenter->g;
prim->vars[vars::B3].eval();
primGuessPlusEps->vars[vars::B1] = prim->vars[vars::B1];
primGuessPlusEps->vars[vars::B2] = prim->vars[vars::B2];
primGuessPlusEps->vars[vars::B3] = prim->vars[vars::B3];
primGuessLineSearchTrial->vars[vars::B1] = prim->vars[vars::B1];
primGuessLineSearchTrial->vars[vars::B2] = prim->vars[vars::B2];
primGuessLineSearchTrial->vars[vars::B3] = prim->vars[vars::B3];
af::sync();
inductionEqnTime = af::timer::stop(inductionEqnTimer);
/* Solve dU/dt + div.F - S = 0 to get prim at n+1/2. NOTE: prim already has
* primHalfStep as a guess */
/* Use simple ideal solver if able and requested */
if (params::conduction == 0 &&
params::viscosity == 0 &&
params::solver == solvers::IDEAL)
{
stepConsTimer = af::timer::start();
timeStepFluidCons(dt);
af::sync();
stepConsTime = af::timer::stop(stepConsTimer);
int numreads, numwrites;
solverTimer = af::timer::start();
idealSolver(*prim, numreads, numwrites);
af::sync();
solverTime = af::timer::stop(solverTimer);
} else {
jacobianAssemblyTime = 0.;
lineSearchTime = 0.;
linearSolverTime = 0.;
solverTimer = af::timer::start();
solve(*prim);
solverTime = af::timer::stop(solverTimer);
}
af::timer fullStepCommTimer = af::timer::start();
/* Copy solution to primOldGhosted */
for (int var=0; var < prim->numVars; var++)
{
primOld->vars[var] = prim->vars[var];
numReads += 1;
numWrites += 1;
}
/* Compute diagnostics */
primOld->communicate();
af::sync();
double fullStepCommTime = af::timer::stop(fullStepCommTimer);
time += dt;
af::timer fullStepDiagTimer = af::timer::start();
fullStepDiagnostics(numReads,numWrites);
af::sync();
double fullStepDiagTime = af::timer::stop(fullStepDiagTimer);
/* done */
double fullStepTime = af::timer::stop(fullStepTimer);
double timeStepTime = af::timer::stop(timeStepTimer);
PetscPrintf(PETSC_COMM_WORLD, "\n");
PetscPrintf(PETSC_COMM_WORLD, " ---Performance report--- \n");
PetscPrintf(PETSC_COMM_WORLD, " Boundary Conditions : %g secs, %g %\n",
boundaryTime, boundaryTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Setting elemHalfStep: %g secs, %g %\n",
elemHalfStepTime,
elemHalfStepTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Explicit sources : %g secs, %g %\n",
explicitSourcesTime,
explicitSourcesTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Implciit sources : %g secs, %g %\n",
implicitSourcesTime,
implicitSourcesTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Divergence of fluxes: %g secs, %g %\n",
divFluxTime, divFluxTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Nonlinear solver : %g secs, %g %\n",
solverTime, solverTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " |- Jacobian assembly: %g secs, %g %\n",
jacobianAssemblyTime,
jacobianAssemblyTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " |- Linear solver : %g secs, %g %\n",
linearSolverTime,
linearSolverTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " |- Linesearch : %g secs, %g %\n",
lineSearchTime,
lineSearchTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Fluid cons : %g secs, %g %\n",
stepConsTime,
stepConsTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Induction equation : %g secs, %g %\n",
inductionEqnTime,
inductionEqnTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Communication : %g secs, %g %\n",
fullStepCommTime,
fullStepCommTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Diagnostics : %g secs, %g %\n",
fullStepDiagTime,
fullStepDiagTime/fullStepTime * 100
);
PetscPrintf(PETSC_COMM_WORLD, " Full step time : %g secs\n\n",
fullStepTime
);
PetscPrintf(PETSC_COMM_WORLD, " ---Performance / proc : %g Zone cycles/sec/proc\n",
prim->N1Local
* prim->N2Local
* prim->N3Local / timeStepTime
);
PetscPrintf(PETSC_COMM_WORLD, " ---Total Performance : %g Zone cycles/sec\n",
N1 * N2 * N3
/ timeStepTime
);
}
