//--------------------------------------------------------------
void WeakEquationElectronMomentum::B_integrand(
const FIELD_MATS &fields,
const GRAD_FIELD_MATS &grad_fields,
const Material * material,
DENS_MAT_VEC &flux) const
{
convection(fields,material,flux);
}
void NSThermalEqResid<EvalT, Traits>::
evaluateFields(typename Traits::EvalData workset)
{
typedef Intrepid2::FunctionSpaceTools FST;
FST::scalarMultiplyDataData<ScalarT> (flux, ThermalCond, TGrad);
FST::integrate<ScalarT>(TResidual, flux, wGradBF, Intrepid2::COMP_CPP, false); // "false" overwrites
for (std::size_t cell=0; cell < workset.numCells; ++cell) {
for (std::size_t qp=0; qp < numQPs; ++qp) {
convection(cell,qp) = 0.0;
if (haveSource) convection(cell,qp) -= Source(cell,qp);
if (workset.transientTerms && enableTransient)
convection(cell,qp) += rho(cell,qp) * Cp(cell,qp) * Tdot(cell,qp);
if (haveFlow) {
for (std::size_t i=0; i < numDims; ++i) {
convection(cell,qp) +=
rho(cell,qp) * Cp(cell,qp) * V(cell,qp,i) * TGrad(cell,qp,i);
}
}
}
}
FST::integrate<ScalarT>(TResidual, convection, wBF, Intrepid2::COMP_CPP, true); // "true" sums into
if (haveSUPG) {
for (std::size_t cell=0; cell < workset.numCells; ++cell) {
for (std::size_t node=0; node < numNodes; ++node) {
for (std::size_t qp=0; qp < numQPs; ++qp) {
for (std::size_t j=0; j < numDims; ++j) {
TResidual(cell,node) +=
rho(cell,qp) * Cp(cell,qp) * TauT(cell,qp) * convection(cell,qp) *
V(cell,qp,j) * wGradBF(cell,node,qp,j);
}
}
}
}
}
}
bool FEMPhysics::eulerian_residual (bool request_jacobian,
DiffContext &/*c*/)
{
// Only calculate a mesh movement residual if it's necessary
if (!_mesh_sys)
return request_jacobian;
libmesh_not_implemented();
#if 0
FEMContext &context = libmesh_cast_ref<FEMContext&>(c);
// This function only supports fully coupled mesh motion for now
libmesh_assert_equal_to (_mesh_sys, this);
unsigned int n_qpoints = (context.get_element_qrule())->n_points();
const unsigned int n_x_dofs = (_mesh_x_var == libMesh::invalid_uint) ?
0 : context.dof_indices_var[_mesh_x_var].size();
const unsigned int n_y_dofs = (_mesh_y_var == libMesh::invalid_uint) ?
0 : context.dof_indices_var[_mesh_y_var].size();
const unsigned int n_z_dofs = (_mesh_z_var == libMesh::invalid_uint) ?
0 : context.dof_indices_var[_mesh_z_var].size();
const unsigned int mesh_xyz_var = n_x_dofs ? _mesh_x_var :
(n_y_dofs ? _mesh_y_var :
(n_z_dofs ? _mesh_z_var :
libMesh::invalid_uint));
// If we're our own _mesh_sys, we'd better be in charge of
// at least one coordinate, and we'd better have the same
// FE type for all coordinates we are in charge of
libmesh_assert_not_equal_to (mesh_xyz_var, libMesh::invalid_uint);
libmesh_assert(!n_x_dofs || context.element_fe_var[_mesh_x_var] ==
context.element_fe_var[mesh_xyz_var]);
libmesh_assert(!n_y_dofs || context.element_fe_var[_mesh_y_var] ==
context.element_fe_var[mesh_xyz_var]);
libmesh_assert(!n_z_dofs || context.element_fe_var[_mesh_z_var] ==
context.element_fe_var[mesh_xyz_var]);
const std::vector<std::vector<Real> > &psi =
context.element_fe_var[mesh_xyz_var]->get_phi();
for (unsigned int var = 0; var != context.n_vars(); ++var)
{
// Mesh motion only affects time-evolving variables
if (this->is_time_evolving(var))
continue;
// The mesh coordinate variables themselves are Lagrangian,
// not Eulerian, and no convective term is desired.
if (/*_mesh_sys == this && */
(var == _mesh_x_var ||
var == _mesh_y_var ||
var == _mesh_z_var))
continue;
// Some of this code currently relies on the assumption that
// we can pull mesh coordinate data from our own system
if (_mesh_sys != this)
libmesh_not_implemented();
// This residual should only be called by unsteady solvers:
// if the mesh is steady, there's no mesh convection term!
UnsteadySolver *unsteady;
if (this->time_solver->is_steady())
return request_jacobian;
else
unsteady = libmesh_cast_ptr<UnsteadySolver*>(this->time_solver.get());
const std::vector<Real> &JxW =
context.element_fe_var[var]->get_JxW();
const std::vector<std::vector<Real> > &phi =
context.element_fe_var[var]->get_phi();
const std::vector<std::vector<RealGradient> > &dphi =
context.element_fe_var[var]->get_dphi();
const unsigned int n_u_dofs = context.dof_indices_var[var].size();
DenseSubVector<Number> &Fu = *context.elem_subresiduals[var];
DenseSubMatrix<Number> &Kuu = *context.elem_subjacobians[var][var];
DenseSubMatrix<Number> *Kux = n_x_dofs ?
context.elem_subjacobians[var][_mesh_x_var] : NULL;
DenseSubMatrix<Number> *Kuy = n_y_dofs ?
context.elem_subjacobians[var][_mesh_y_var] : NULL;
DenseSubMatrix<Number> *Kuz = n_z_dofs ?
context.elem_subjacobians[var][_mesh_z_var] : NULL;
std::vector<Real> delta_x(n_x_dofs, 0.);
std::vector<Real> delta_y(n_y_dofs, 0.);
std::vector<Real> delta_z(n_z_dofs, 0.);
for (unsigned int i = 0; i != n_x_dofs; ++i)
{
unsigned int j = context.dof_indices_var[_mesh_x_var][i];
delta_x[i] = libmesh_real(this->current_solution(j)) -
libmesh_real(unsteady->old_nonlinear_solution(j));
}
for (unsigned int i = 0; i != n_y_dofs; ++i)
{
unsigned int j = context.dof_indices_var[_mesh_y_var][i];
delta_y[i] = libmesh_real(this->current_solution(j)) -
libmesh_real(unsteady->old_nonlinear_solution(j));
}
for (unsigned int i = 0; i != n_z_dofs; ++i)
{
unsigned int j = context.dof_indices_var[_mesh_z_var][i];
delta_z[i] = libmesh_real(this->current_solution(j)) -
libmesh_real(unsteady->old_nonlinear_solution(j));
}
for (unsigned int qp = 0; qp != n_qpoints; ++qp)
{
Gradient grad_u = context.interior_gradient(var, qp);
RealGradient convection(0.);
for (unsigned int i = 0; i != n_x_dofs; ++i)
convection(0) += delta_x[i] * psi[i][qp];
for (unsigned int i = 0; i != n_y_dofs; ++i)
convection(1) += delta_y[i] * psi[i][qp];
for (unsigned int i = 0; i != n_z_dofs; ++i)
convection(2) += delta_z[i] * psi[i][qp];
for (unsigned int i = 0; i != n_u_dofs; ++i)
{
Number JxWxPhiI = JxW[qp] * phi[i][qp];
Fu(i) += (convection * grad_u) * JxWxPhiI;
if (request_jacobian)
{
Number JxWxPhiI = JxW[qp] * phi[i][qp];
for (unsigned int j = 0; j != n_u_dofs; ++j)
Kuu(i,j) += JxWxPhiI * (convection * dphi[j][qp]);
Number JxWxPhiIoverDT = JxWxPhiI/this->deltat;
Number JxWxPhiIxDUDXoverDT = JxWxPhiIoverDT * grad_u(0);
for (unsigned int j = 0; j != n_x_dofs; ++j)
(*Kux)(i,j) += JxWxPhiIxDUDXoverDT * psi[j][qp];
Number JxWxPhiIxDUDYoverDT = JxWxPhiIoverDT * grad_u(1);
for (unsigned int j = 0; j != n_y_dofs; ++j)
(*Kuy)(i,j) += JxWxPhiIxDUDYoverDT * psi[j][qp];
Number JxWxPhiIxDUDZoverDT = JxWxPhiIoverDT * grad_u(2);
for (unsigned int j = 0; j != n_z_dofs; ++j)
(*Kuz)(i,j) += JxWxPhiIxDUDZoverDT * psi[j][qp];
}
}
}
}
#endif // 0
return request_jacobian;
}
//------------------------------------------------------------------------------
int main( int argc, char * argv[] )
{
//--------------------------------------------------------------------------
const unsigned geomDeg = 1;
const unsigned dim = 2;
// degrees of lowest-order TH element
const unsigned fieldDegU = 2;
const unsigned fieldDegP = 1;
const unsigned tiOrder = 1;
typedef base::time::BDF<tiOrder> MSM;
const base::Shape shape = base::SimplexShape<dim>::value;
const base::Shape surfShape = base::SimplexShape<dim-1>::value;
//--------------------------------------------------------------------------
if ( argc != 2 ) {
std::cout << "Usage: " << argv[0] << " input.dat \n\n";
return -1;
}
const std::string inputFile = boost::lexical_cast<std::string>( argv[1] );
//--------------------------------------------------------------------------
std::string meshFile, surfFile;
double viscosity, density, tolerance, penaltyFactor, stepSize;
unsigned maxIter, numSteps;
{
//Feed properties parser with the variables to be read
base::io::PropertiesParser prop;
prop.registerPropertiesVar( "meshFile", meshFile );
prop.registerPropertiesVar( "surfFile", surfFile );
prop.registerPropertiesVar( "viscosity", viscosity );
prop.registerPropertiesVar( "density", density );
prop.registerPropertiesVar( "maxIter", maxIter );
prop.registerPropertiesVar( "tolerance", tolerance );
prop.registerPropertiesVar( "penaltyFactor", penaltyFactor );
prop.registerPropertiesVar( "stepSize", stepSize );
prop.registerPropertiesVar( "numSteps", numSteps );
// Read variables from the input file
std::ifstream inp( inputFile.c_str() );
VERIFY_MSG( inp.is_open(), "Cannot open input file" );
prop.readValues( inp );
inp.close( );
// Make sure all variables have been found
if ( not prop.isEverythingRead() ) {
prop.writeUnread( std::cerr );
VERIFY_MSG( false, "Could not find above variables" );
}
}
const std::string baseName = base::io::baseName( meshFile, ".smf" );
//--------------------------------------------------------------------------
typedef base::Unstructured<shape,geomDeg> Mesh;
Mesh mesh;
{
std::ifstream smf( meshFile.c_str() );
VERIFY_MSG( smf.is_open(), "Cannot open mesh file" );
base::io::smf::readMesh( smf, mesh );
smf.close();
}
//--------------------------------------------------------------------------
// Surface mesh
typedef base::Unstructured<surfShape,1,dim> SurfMesh;
SurfMesh surfMesh;
{
std::ifstream smf( surfFile.c_str() );
base::io::smf::readMesh( smf, surfMesh );
smf.close();
}
//--------------------------------------------------------------------------
// Compute the level set data
typedef base::cut::LevelSet<dim> LevelSet;
std::vector<LevelSet> levelSet;
const bool isSigned = true;
base::cut::bruteForce( mesh, surfMesh, isSigned, levelSet );
const unsigned kernelDegEstimate = 5;
//--------------------------------------------------------------------------
// Make cut cell structure
typedef base::cut::Cell<shape> Cell;
std::vector<Cell> cells;
base::cut::generateCutCells( mesh, levelSet, cells );
// Quadrature
typedef base::cut::Quadrature<kernelDegEstimate,shape> CutQuadrature;
CutQuadrature quadrature( cells, true );
// for surface
typedef base::SurfaceQuadrature<kernelDegEstimate,shape> SurfaceQuadrature;
SurfaceQuadrature surfaceQuadrature;
//------------------------------------------------------------------------------
// Finite element bases
const unsigned nHist = MSM::numSteps;
const unsigned doFSizeU = dim;
typedef base::fe::Basis<shape,fieldDegU> FEBasisU;
typedef base::cut::ScaledField<FEBasisU,doFSizeU,nHist> Velocity;
Velocity velocity;
base::dof::generate<FEBasisU>( mesh, velocity );
const unsigned doFSizeP = 1;
typedef base::fe::Basis<shape,fieldDegP> FEBasisP;
typedef base::cut::ScaledField<FEBasisP,doFSizeP,nHist> Pressure;
Pressure pressure;
base::dof::generate<FEBasisP>( mesh, pressure );
const unsigned doFSizeS = dim;
typedef base::fe::Basis<surfShape,1> FEBasisS;
typedef base::Field<FEBasisS,doFSizeS> SurfField;
SurfField surfVelocity, surfForces;
base::dof::generate<FEBasisS>( surfMesh, surfVelocity );
base::dof::generate<FEBasisS>( surfMesh, surfForces );
// set initial condition to the identity
base::dof::setField( surfMesh, surfVelocity,
boost::bind( &surfaceVelocity<dim,
SurfField::DegreeOfFreedom>, _1, _2 ) );
// boundary datum
base::cut::TransferSurfaceDatum<SurfMesh,SurfField,Mesh::Element>
s2d( surfMesh, surfVelocity, levelSet );
//--------------------------------------------------------------------------
// surface mesh
typedef base::mesh::BoundaryMeshBinder<Mesh>::Type BoundaryMesh;
BoundaryMesh boundaryMesh, immersedMesh;
// from boundary
{
// identify list of element boundary faces
base::mesh::MeshBoundary meshBoundary;
meshBoundary.create( mesh.elementsBegin(), mesh.elementsEnd() );
// generate a mesh from that list (with a filter)
base::mesh::generateBoundaryMesh( meshBoundary.begin(),
meshBoundary.end(),
mesh, boundaryMesh,
boost::bind( &boundaryFilter<dim>, _1 ) );
// make a surface mesh from the implicit surface
base::cut::generateSurfaceMesh<Mesh,Cell>( mesh, cells, immersedMesh );
}
// the composite field with geometry, velocity and pressure
typedef base::asmb::FieldBinder<Mesh,Velocity,Pressure> Field;
Field field( mesh, velocity, pressure );
// define the system blocks (U,U), (U,P), and (P,U)
typedef Field::TupleBinder<1,1,1>::Type TopLeft;
typedef Field::TupleBinder<1,2>::Type TopRight;
typedef Field::TupleBinder<2,1>::Type BotLeft;
std::vector<double> supportsU, supportsP;
std::size_t numDoFsU = std::distance( velocity.doFsBegin(), velocity.doFsEnd() );
supportsU.resize( numDoFsU );
std::size_t numDoFsP = std::distance( pressure.doFsBegin(), pressure.doFsEnd() );
supportsP.resize( numDoFsP );
base::cut::supportComputation( mesh, velocity, quadrature, supportsU );
base::cut::supportComputation( mesh, pressure, quadrature, supportsP );
velocity.scaleAndTagBasis( supportsU, 1.e-8 );
pressure.scaleAndTagBasis( supportsP, 1.e-8 );
//velocity.tagBasis( supportsU, 1.e-8 );
//pressure.tagBasis( supportsP, 1.e-8 );
// Fix one pressure dof
// Pressure::DoFPtrIter pIter = pressure.doFsBegin();
// std::advance( pIter, std::distance( pressure.doFsBegin(), pressure.doFsEnd() )/5 );
// (*pIter) -> constrainValue( 0, 0.0 );
// Number of DoFs after constraint application!
numDoFsU =
base::dof::numberDoFsConsecutively( velocity.doFsBegin(), velocity.doFsEnd() );
std::cout << "# Number of velocity dofs " << numDoFsU << std::endl;
numDoFsP =
base::dof::numberDoFsConsecutively( pressure.doFsBegin(), pressure.doFsEnd(),
numDoFsU );
std::cout << "# Number of pressure dofs " << numDoFsP << std::endl;
// kernels
typedef fluid::StressDivergence< TopLeft::Tuple> StressDivergence;
typedef fluid::Convection< TopLeft::Tuple> Convection;
typedef fluid::PressureGradient< TopRight::Tuple> GradP;
typedef fluid::VelocityDivergence<BotLeft::Tuple> DivU;
StressDivergence stressDivergence( viscosity );
Convection convection( density );
GradP gradP;
DivU divU( true );
// for surface fields
typedef base::asmb::SurfaceFieldBinder<BoundaryMesh,Velocity,Pressure> SurfaceFieldBinder;
typedef SurfaceFieldBinder::TupleBinder<1,1,1>::Type STBUU;
typedef SurfaceFieldBinder::TupleBinder<1,2 >::Type STBUP;
SurfaceFieldBinder boundaryFieldBinder( boundaryMesh, velocity, pressure );
SurfaceFieldBinder immersedFieldBinder( immersedMesh, velocity, pressure );
std::ofstream forces( "forces.dat" );
for ( unsigned step = 0; step < numSteps; step++ ) {
const double time = step * stepSize;
const double factor = ( time < 1.0 ? time : 1.0 );
std::cout << step << ": time=" << time << ", factor=" << factor
<< "\n";
//base::dof::clearDoFs( velocity );
//base::dof::clearDoFs( pressure );
//--------------------------------------------------------------------------
// Nonlinear iterations
unsigned iter = 0;
while( iter < maxIter ) {
// Create a solver object
typedef base::solver::Eigen3 Solver;
Solver solver( numDoFsU + numDoFsP );
std::cout << "* Iteration " << iter << std::flush;
// compute inertia terms, d/dt, due to time integration
base::time::computeInertiaTerms<TopLeft,MSM>( quadrature, solver,
field, stepSize, step,
density );
// Compute system matrix
base::asmb::stiffnessMatrixComputation<TopLeft>( quadrature, solver,
field, stressDivergence );
base::asmb::stiffnessMatrixComputation<TopLeft>( quadrature, solver,
field, convection );
base::asmb::stiffnessMatrixComputation<TopRight>( quadrature, solver,
field, gradP );
base::asmb::stiffnessMatrixComputation<BotLeft>( quadrature, solver,
field, divU );
// compute residual forces
base::asmb::computeResidualForces<TopLeft >( quadrature, solver, field,
stressDivergence );
base::asmb::computeResidualForces<TopLeft >( quadrature, solver, field, convection );
base::asmb::computeResidualForces<TopRight>( quadrature, solver, field, gradP );
base::asmb::computeResidualForces<BotLeft >( quadrature, solver, field, divU );
// Parameter classes
base::nitsche::OuterBoundary ob( viscosity );
base::nitsche::ImmersedBoundary<Cell> ib( viscosity, cells );
// Penalty method
base::nitsche::penaltyLHS<STBUU>( surfaceQuadrature, solver,
boundaryFieldBinder, ob, penaltyFactor );
base::nitsche::penaltyRHS<STBUU>( surfaceQuadrature, solver, boundaryFieldBinder,
boost::bind( &dirichlet<dim>, _1, factor),
ob, penaltyFactor );
base::nitsche::penaltyLHS<STBUU>( surfaceQuadrature, solver,
immersedFieldBinder, ib, penaltyFactor );
base::nitsche::penaltyRHS2<STBUU>( surfaceQuadrature, solver, immersedFieldBinder,
s2d, ib, penaltyFactor );
// Nitsche terms
base::nitsche::primalEnergyLHS<STBUU>( stressDivergence, surfaceQuadrature, solver,
boundaryFieldBinder, ob );
base::nitsche::dualEnergyLHS<STBUU>( stressDivergence, surfaceQuadrature, solver,
boundaryFieldBinder, ob );
base::nitsche::energyRHS<STBUU>( stressDivergence, surfaceQuadrature, solver,
boundaryFieldBinder,
boost::bind( &dirichlet<dim>, _1, factor),
ob );
base::nitsche::primalEnergyLHS<STBUP>( gradP, surfaceQuadrature, solver,
boundaryFieldBinder, ob );
base::nitsche::dualEnergyLHS<STBUP>( gradP, surfaceQuadrature, solver,
boundaryFieldBinder, ob );
base::nitsche::energyRHS<STBUP>( gradP, surfaceQuadrature, solver,
boundaryFieldBinder,
boost::bind( &dirichlet<dim>, _1, factor), ob );
base::nitsche::energyResidual<STBUU>( stressDivergence, surfaceQuadrature, solver,
boundaryFieldBinder, ob );
base::nitsche::energyResidual<STBUP>( gradP, surfaceQuadrature, solver,
boundaryFieldBinder, ob );
base::nitsche::primalEnergyLHS<STBUU>( stressDivergence, surfaceQuadrature, solver,
immersedFieldBinder, ib );
base::nitsche::dualEnergyLHS<STBUU>( stressDivergence, surfaceQuadrature, solver,
immersedFieldBinder, ib );
base::nitsche::energyRHS2<STBUU>( stressDivergence, surfaceQuadrature, solver,
immersedFieldBinder, s2d, ib );
base::nitsche::primalEnergyLHS<STBUP>( gradP, surfaceQuadrature, solver,
immersedFieldBinder, ib );
base::nitsche::dualEnergyLHS<STBUP>( gradP, surfaceQuadrature, solver,
immersedFieldBinder, ib );
base::nitsche::energyRHS2<STBUP>( gradP, surfaceQuadrature, solver,
immersedFieldBinder, s2d, ib );
base::nitsche::energyResidual<STBUU>( stressDivergence, surfaceQuadrature, solver,
immersedFieldBinder, ib );
base::nitsche::energyResidual<STBUP>( gradP, surfaceQuadrature, solver,
immersedFieldBinder, ib );
// Finalise assembly
solver.finishAssembly();
// check convergence via solver norms
const double residualNorm = solver.norm();
std::cout << " |R| = " << residualNorm << std::flush;
if ( residualNorm < tolerance * viscosity) {
std::cout << std::endl;
break;
}
// Solve
solver.superLUSolve();
// distribute results back to dofs
base::dof::addToDoFsFromSolver( solver, velocity );
base::dof::addToDoFsFromSolver( solver, pressure );
//base::dof::setDoFsFromSolver( solver, pressure );
// check convergence via solver norms
const double incrementNorm = solver.norm(0, numDoFsU );
std::cout << " |dU| = " << incrementNorm << std::endl;
// push history
base::dof::pushHistory( velocity );
base::dof::pushHistory( pressure );
if ( incrementNorm < tolerance ) break;
iter++;
}
writeVTKFile( baseName, step, mesh, velocity, pressure, levelSet, viscosity );
{
base::Vector<dim>::Type sumOfForces = base::constantVector<dim>( 0. );
typedef Field::TupleBinder<1,2>::Type UP;
//typedef fluid::Stress<UP::Tuple> Stress;
//Stress stress( viscosity );
typedef fluid::Traction<UP::Tuple> Traction;
Traction traction( viscosity );
base::cut::ComputeSurfaceForces<SurfMesh,SurfField,
SurfaceQuadrature,STBUP::Tuple,Traction>
computeSurfaceForces( surfMesh, surfForces, surfaceQuadrature, levelSet, traction );
SurfaceFieldBinder::FieldIterator first = immersedFieldBinder.elementsBegin();
SurfaceFieldBinder::FieldIterator last = immersedFieldBinder.elementsEnd();
for ( ; first != last; ++first ) {
sumOfForces +=
computeSurfaceForces( STBUP::makeTuple( *first ) );
}
writeSurfaceVTKFile( baseName, step, surfMesh, surfVelocity, surfForces );
std::cout << " F= " << sumOfForces.transpose() << " \n";
forces << time << " " << sumOfForces.transpose() << std::endl;;
}
}
forces.close();
return 0;
}
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);
int commsize, commrank;
MPI_Comm_size(MPI_COMM_WORLD, &commsize);
MPI_Comm_rank(MPI_COMM_WORLD, &commrank);
const bool isRoot = commrank == 0;
const bool isLast = commrank == commsize - 1;
if (isRoot)
std::cout << "Initialization...\n" << std::endl;
RuntimeConfiguration conf(argc, argv);
// Compute my start and end time
const double timeStart = conf.timeSliceSize() * commrank;
const double timeEnd = conf.timeSliceSize() * (commrank + 1);
if (isRoot)
std::cout << "Running with:\n"
<< " - initial diffusion coefficient: " << conf.nu0() << "\n"
<< " - frequence of diffusion coefficient: " << conf.nufreq() << "\n"
<< " - advection velocity in x: " << conf.cx() << "\n"
<< " - advection velocity in y: " << conf.cy() << "\n"
<< " - advection velocity in z: " << conf.cz() << "\n"
<< " - spatial discretization step: " << conf.dx() << "\n"
<< " - endtime: " << conf.endTime() << "\n"
<< " - number of time slices: " << conf.timeSlices() << "\n"
<< " - time slice size: " << conf.timeSliceSize() << "\n"
<< " - CFL fine: " << conf.cflFine() << "\n"
<< " - CFL coarse: " << conf.cflCoarse() << "\n"
<< " - timestep size fine: " << conf.dtFine() << "\n"
<< " - timestep size coarse: " << conf.dtCoarse() << "\n"
<< " - timesteps per slice fine propagator: " << conf.timeStepsFinePerTimeSlice() << "\n"
<< " - timesteps per slice coarse propagator: " << conf.timeStepsCoarsePerTimeSlice() << "\n"
<< " - parareal iterations: " << conf.kmax() << "\n"
<< " - asynchronous communications: " << (conf.async() ? "Enabled" : "Disabled") << "\n"
<< " - intermediate fields in mat files: " << (conf.mat() ? "Yes" : "No") << "\n"
<< std::endl;
// Calculation domain and boundaries
IJKSize domain; domain.Init(conf.gridSize(), conf.gridSize(), conf.gridSize());
KBoundary kboundary; kboundary.Init(-convectionBoundaryLines, convectionBoundaryLines);
// Initialize fields
ConvectionField q, qinitial;
q.Init("q", domain, kboundary);
qinitial.Init("qinitial", domain, kboundary);
Convection convection(conf.gridSize(), conf.gridSize(), conf.gridSize(), conf.dx(), conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz());
// Initialize parareal
Parareal<Convection, ConvectionField> parareal(convection, qinitial, q, timeStart, conf, MPI_COMM_WORLD);
if (conf.mode() == ModeCompare)
{
// Measure time required by convection
const int tauSamples = 4;
double tauF = MPI_Wtime();
convection.DoRK4(qinitial, qinitial, 0., conf.dtFine(), tauSamples*conf.timeStepsFinePerTimeSlice());
SynchronizeCUDA();
tauF = MPI_Wtime() - tauF;
double tauG = MPI_Wtime();
convection.DoEuler(qinitial, qinitial, 0., conf.dtCoarse(), tauSamples*conf.timeStepsCoarsePerTimeSlice());
SynchronizeCUDA();
tauG = MPI_Wtime() - tauG;
const double tauRatio = tauG / tauF;
const double Nit_Np = static_cast<double>(conf.kmax()) / commsize;
const double maxSpeedup = 1. / (tauRatio * (1. + Nit_Np) + Nit_Np);
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
MPI_Barrier(MPI_COMM_WORLD);
double eserial = MPI_Wtime();
parareal.DoSerial();
eserial = MPI_Wtime() - eserial;
// Save reference
ConvectionField qreference = q;
SynchronizeHost(qreference);
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
MPI_Barrier(MPI_COMM_WORLD);
double eparallel = MPI_Wtime();
parareal.DoParallel();
eparallel = MPI_Wtime() - eparallel;
// Output
MPI_Barrier(MPI_COMM_WORLD);
if (isLast)
{
double e = computeErrorReference(q, qreference);
std::cout << "\n"
<< "Serial run time: " << eserial << "\n"
<< "Parallel run time: " << eparallel << "\n"
<< "Speedup: " << eserial / eparallel << "\n"
<< "Maximal speedup: " << maxSpeedup << "\n"
<< "Error at end: " << e << "\n"
<< std::endl;
MatFile matfile("result.mat");
matfile.addField("q", q);
matfile.addField("qreference", qreference);
}
}
else if (conf.mode() == ModeSerial)
{
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
double e = MPI_Wtime();
double energyStart = energy();
double deviceEnergyStart = deviceEnergy();
parareal.DoSerial();
e = MPI_Wtime() - e;
double energyEnd = energy();
double deviceEnergyEnd = deviceEnergy();
const double totDevice = totalEnergy(deviceEnergyStart, deviceEnergyEnd);
const double totNode = totalEnergy(energyStart, energyEnd) - totDevice;
const double totNetwork = e * powerNetwork;
const double totBlower = e * powerBlower;
const double totEnergy = totNode + totDevice + totNetwork + totBlower;
// Output
MPI_Barrier(MPI_COMM_WORLD);
if (isLast)
{
std::cout << "\n" << "Serial run time: " << e << "\n";
std::printf("Node energy : %8f J (%8.3e W/node)\n", totNode , totNode/e);
std::printf("Device energy : %8f J (%8.3e W/node)\n", totDevice , totDevice/e);
std::printf("Network energy: %8f J (%8.3e W/node)\n", totNetwork, totNetwork/e);
std::printf("Blower energy : %8f J (%8.3e W/node)\n", totBlower , totBlower/e);
std::printf("Total energy : %8f J (%8.3e W/node)\n", totEnergy , totEnergy/e);
std::cout << std::endl;
}
}
else if (conf.mode() == ModeParallel)
{
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
parareal.DoSerial();
std::cout << " -- The serial computation is done\n";
ConvectionField qreference = q;
// Run parallel
MPI_Barrier(MPI_COMM_WORLD);
double e = MPI_Wtime();
double energyStart = energy();
double deviceEnergyStart = deviceEnergy();
parareal.DoParallel();
MPI_Barrier(MPI_COMM_WORLD);
e = MPI_Wtime() - e;
std::cout << " -- The parallel computation is done\n";
double energyEnd = energy();
double deviceEnergyEnd = deviceEnergy();
const double totDevice = totalEnergy(deviceEnergyStart, deviceEnergyEnd, MPI_COMM_WORLD);
const double totNode = totalEnergy(energyStart, energyEnd, MPI_COMM_WORLD) - totDevice;
const double totNetwork = e * powerNetwork * commsize;
const double totBlower = e * powerBlower * commsize;
const double totEnergy = totNode + totDevice + totNetwork + totBlower;
// Compute error
double error = computeErrorReference(q, qreference);
// Output
MPI_Barrier(MPI_COMM_WORLD);
if (isLast)
{
const double fac = 1./e/commsize;
std::cout << std::endl;
std::printf("Parallel run time: %f s\n", e);
std::printf("Node energy : %8f J (%8.3e W/node)\n", totNode , fac*totNode);
std::printf("Device energy : %8f J (%8.3e W/node)\n", totDevice , fac*totDevice);
std::printf("Network energy: %8f J (%8.3e W/node)\n", totNetwork, fac*totNetwork);
std::printf("Blower energy : %8f J (%8.3e W/node)\n", totBlower , fac*totBlower);
std::printf("Total energy : %8f J (%8.3e W/node)\n", totEnergy , fac*totEnergy);
std::printf("Error of parareal: %.4e\n", error);
std::cout << std::endl;
}
}
else if (conf.mode() == ModeTiming)
{
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
std::vector<double> times;
MPI_Barrier(MPI_COMM_WORLD);
parareal.DoTimedParallel(times);
// Gather on root
const int s = times.size();
std::vector<double> timesGlobal;
timesGlobal.resize(s * commsize);
MPI_Gather(×[0], s, MPI_DOUBLE, ×Global[0], s, MPI_DOUBLE, 0, MPI_COMM_WORLD);
// Output
if (isRoot)
{
std::cout << "\nTimes:\n";
for(int i = 0; i < s; ++i)
{
for(int p = 0; p < commsize; ++p)
{
std::cout << std::scientific << std::setprecision(6) << timesGlobal[p*s + i] << " ";
}
std::cout << "\n";
}
}
}
// Finalize
MPI_Finalize();
return 0;
}
void HeatEqResid<EvalT, Traits>::
evaluateFields(typename Traits::EvalData workset)
{
//// workset.print(std::cout);
typedef Intrepid::FunctionSpaceTools FST;
// Since Intrepid will later perform calculations on the entire workset size
// and not just the used portion, we must fill the excess with reasonable
// values. Leaving this out leads to floating point exceptions !!!
for (std::size_t cell=workset.numCells; cell < worksetSize; ++cell)
for (std::size_t qp=0; qp < numQPs; ++qp){
ThermalCond(cell,qp) = 0.0;
for (std::size_t i=0; i < numDims; ++i){
flux(cell,qp,i) = 0.0;
TGrad(cell,qp,i) = 0.0;
}
}
FST::scalarMultiplyDataData<ScalarT> (flux, ThermalCond, TGrad);
FST::integrate<ScalarT>(TResidual, flux, wGradBF, Intrepid::COMP_CPP, false); // "false" overwrites
if (haveSource) {
for (std::size_t cell=workset.numCells; cell < worksetSize; ++cell)
for (std::size_t qp=0; qp < numQPs; ++qp)
Source(cell,qp) = 0.0;
for (int i =0; i< Source.dimension(0); i++)
for (int j =0; j< Source.dimension(1); j++)
Source(i,j) *= -1.0;
FST::integrate<ScalarT>(TResidual, Source, wBF, Intrepid::COMP_CPP, true); // "true" sums into
}
if (workset.transientTerms && enableTransient){
for (std::size_t cell=workset.numCells; cell < worksetSize; ++cell)
for (std::size_t qp=0; qp < numQPs; ++qp)
Tdot(cell,qp) = 0.0;
FST::integrate<ScalarT>(TResidual, Tdot, wBF, Intrepid::COMP_CPP, true); // "true" sums into
}
if (haveConvection) {
Intrepid::FieldContainer<ScalarT> convection(worksetSize, numQPs);
for (std::size_t cell=workset.numCells; cell < worksetSize; ++cell)
for (std::size_t qp=0; qp < numQPs; ++qp)
convection(cell,qp) = 0.0;
for (std::size_t cell=0; cell < workset.numCells; ++cell) {
for (std::size_t qp=0; qp < numQPs; ++qp) {
convection(cell,qp) = 0.0;
for (std::size_t i=0; i < numDims; ++i) {
if (haverhoCp)
convection(cell,qp) += rhoCp(cell,qp) * convectionVels[i] * TGrad(cell,qp,i);
else
convection(cell,qp) += convectionVels[i] * TGrad(cell,qp,i);
}
}
}
FST::integrate<ScalarT>(TResidual, convection, wBF, Intrepid::COMP_CPP, true); // "true" sums into
}
if (haveAbsorption) {
// Since Intrepid will later perform calculations on the entire workset size
// and not just the used portion, we must fill the excess with reasonable
// values. Leaving this out leads to floating point exceptions !!!
for (std::size_t cell=workset.numCells; cell < worksetSize; ++cell)
for (std::size_t qp=0; qp < numQPs; ++qp){
aterm(cell,qp) = 0.0;
Absorption(cell,qp) = 0.0;
Temperature(cell,qp) = 0.0;
}
FST::scalarMultiplyDataData<ScalarT> (aterm, Absorption, Temperature);
FST::integrate<ScalarT>(TResidual, aterm, wBF, Intrepid::COMP_CPP, true);
}
//TResidual.print(std::cout, true);
}
