void average_face_to_cellcenter (MultiFab& cc, const PArray<MultiFab>& fc, const Geometry& geom)
{
BL_ASSERT(cc.nComp() >= BL_SPACEDIM);
BL_ASSERT(fc.size() == BL_SPACEDIM);
BL_ASSERT(fc[0].nComp() == 1); // We only expect fc to have the gradient perpendicular to the face
const Real* dx = geom.CellSize();
const Real* problo = geom.ProbLo();
int coord_type = Geometry::Coord();
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter mfi(cc,true); mfi.isValid(); ++mfi)
{
const Box& bx = mfi.tilebox();
BL_FORT_PROC_CALL(BL_AVG_FC_TO_CC,bl_avg_fc_to_cc)
(bx.loVect(), bx.hiVect(),
BL_TO_FORTRAN(cc[mfi]),
D_DECL(BL_TO_FORTRAN(fc[0][mfi]),
BL_TO_FORTRAN(fc[1][mfi]),
BL_TO_FORTRAN(fc[2][mfi])),
dx, problo, coord_type);
}
}
void
FMultiGrid::Copy (PArray<MultiFab>& dst, PArray<MultiFab>& src)
{
int nlevels = src.size();
dst.resize(nlevels);
for (int ilev = 0; ilev < nlevels; ++ilev) {
dst.set(ilev, &src[ilev]);
}
}
void
FMultiGrid::Copy (Array<PArray<MultiFab> >& dst, PArray<MultiFab>& src)
{
int ndim = src.size();
dst.resize(1);
dst[0].resize(ndim);
for(int idim = 0; idim < ndim; ++idim) {
dst[0].set(idim, &src[idim]);
}
}
void
FMultiGrid::set_mac_coeffs (PArray<MultiFab>& b)
{
BL_ASSERT(m_coeff.eq_type == invalid_eq);
BL_ASSERT(m_nlevels == 1);
BL_ASSERT(b.size() == BL_SPACEDIM);
m_coeff.eq_type = macproj_eq;
m_coeff.coeffs_set = true;
Copy(m_coeff.b, b);
}
std::string Variable::printArray(PArray array, std::string indent)
{
std::ostringstream result;
result << indent << "(Array length=" << array->size() << ")" << std::endl << indent << "{" << std::endl;
std::string currentIndent = indent;
currentIndent.push_back(' ');
currentIndent.push_back(' ');
for(std::vector<std::shared_ptr<Variable>>::iterator i = array->begin(); i != array->end(); ++i)
{
result << print(*i, currentIndent);
}
result << indent << "}" << std::endl;
return result.str();
}
void
FMultiGrid::set_coefficients (PArray<MultiFab>& a, Array<PArray<MultiFab> > & b)
{
BL_ASSERT(m_coeff.eq_type == invalid_eq || m_coeff.eq_type == general_eq);
BL_ASSERT(!m_coeff.coeffs_set);
BL_ASSERT(m_nlevels == a.size() && m_nlevels == b.size());
BL_ASSERT(BL_SPACEDIM == b[0].size());
m_coeff.eq_type = general_eq;
m_coeff.coeffs_set = true;
Copy(m_coeff.a, a);
Copy(m_coeff.b, b);
}
// Average fine face-based MultiFab onto crse fine-centered MultiFab.
// This routine assumes that the crse BoxArray is a coarsened version of the fine BoxArray.
void average_down_faces (PArray<MultiFab>& fine, PArray<MultiFab>& crse, IntVect& ratio)
{
BL_ASSERT(crse.size() == BL_SPACEDIM);
BL_ASSERT(fine.size() == BL_SPACEDIM);
BL_ASSERT(crse[0].nComp() == 1);
BL_ASSERT(fine[0].nComp() == 1);
#ifdef _OPENMP
#pragma omp parallel
#endif
for (int n=0; n<BL_SPACEDIM; ++n) {
for (MFIter mfi(crse[n],true); mfi.isValid(); ++mfi)
{
const Box& tbx = mfi.tilebox();
BL_FORT_PROC_CALL(BL_AVGDOWN_FACES,bl_avgdown_faces)
(tbx.loVect(),tbx.hiVect(),
BL_TO_FORTRAN(fine[n][mfi]),
BL_TO_FORTRAN(crse[n][mfi]),
ratio.getVect(),n);
}
}
}
FMultiGrid::FMultiGrid (const PArray<Geometry> & geom,
int baselevel,
IntVect crse_ratio)
:
m_nlevels(geom.size()),
m_baselevel(baselevel),
m_crse_ratio(crse_ratio),
m_stencil(CC_CROSS_STENCIL),
m_maxorder(0),
m_verbose(0),
m_geom(m_nlevels),
m_bndry(0),
m_mgt_solver(0)
{
for (int ilev = 0; ilev < m_nlevels; ++ilev) {
m_geom[ilev] = geom[ilev];
}
if (m_baselevel > 0 && m_crse_ratio == IntVect::TheZeroVector())
BoxLib::Abort("FMultiGrid: must set crse_ratio if baselevel > 0");
}
void average_cellcenter_to_face (PArray<MultiFab>& fc, const MultiFab& cc, const Geometry& geom)
{
BL_ASSERT(cc.nComp() == 1);
BL_ASSERT(cc.nGrow() >= 1);
BL_ASSERT(fc.size() == BL_SPACEDIM);
BL_ASSERT(fc[0].nComp() == 1); // We only expect fc to have the gradient perpendicular to the face
const Real* dx = geom.CellSize();
const Real* problo = geom.ProbLo();
int coord_type = Geometry::Coord();
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter mfi(cc,true); mfi.isValid(); ++mfi)
{
const Box& xbx = mfi.nodaltilebox(0);
#if (BL_SPACEDIM > 1)
const Box& ybx = mfi.nodaltilebox(1);
#endif
#if (BL_SPACEDIM == 3)
const Box& zbx = mfi.nodaltilebox(2);
#endif
BL_FORT_PROC_CALL(BL_AVG_CC_TO_FC,bl_avg_cc_to_fc)
(xbx.loVect(), xbx.hiVect(),
#if (BL_SPACEDIM > 1)
ybx.loVect(), ybx.hiVect(),
#endif
#if (BL_SPACEDIM == 3)
zbx.loVect(), zbx.hiVect(),
#endif
D_DECL(BL_TO_FORTRAN(fc[0][mfi]),
BL_TO_FORTRAN(fc[1][mfi]),
BL_TO_FORTRAN(fc[2][mfi])),
BL_TO_FORTRAN(cc[mfi]),
dx, problo, coord_type);
}
}
void
solve_for_accel(PArray<MultiFab>& rhs, PArray<MultiFab>& phi, PArray<MultiFab>& grad_phi,
const Array<Geometry>& geom, int base_level, int finest_level, Real offset)
{
Real tol = 1.e-10;
Real abs_tol = 1.e-14;
Array< PArray<MultiFab> > grad_phi_edge;
grad_phi_edge.resize(rhs.size());
for (int lev = base_level; lev <= finest_level ; lev++)
{
grad_phi_edge[lev].resize(BL_SPACEDIM, PArrayManage);
for (int n = 0; n < BL_SPACEDIM; ++n)
grad_phi_edge[lev].set(n, new MultiFab(BoxArray(rhs[lev].boxArray()).surroundingNodes(n), 1, 1));
}
Real strt = ParallelDescriptor::second();
// ***************************************************
// Make sure the RHS sums to 0 if fully periodic
// ***************************************************
for (int lev = base_level; lev <= finest_level; lev++) {
Real n0 = rhs[lev].norm0();
if (ParallelDescriptor::IOProcessor())
std::cout << "Max of rhs in solve_for_phi before correction at level "
<< lev << " " << n0 << std::endl;
}
for (int lev = base_level; lev <= finest_level; lev++)
rhs[lev].plus(-offset, 0, 1, 0);
for (int lev = base_level; lev <= finest_level; lev++) {
Real n0 = rhs[lev].norm0();
if (ParallelDescriptor::IOProcessor())
std::cout << "Max of rhs in solve_for_phi after correction at level "
<< lev << " " << n0 << std::endl;
}
// ***************************************************
// Solve for phi and return both phi and grad_phi_edge
// ***************************************************
#ifdef USEHPGMG
solve_with_hpgmg(rhs,phi,grad_phi_edge,geom,base_level,finest_level,tol,abs_tol);
#else
solve_with_f90 (rhs,phi,grad_phi_edge,geom,base_level,finest_level,tol,abs_tol);
#endif
// Average edge-centered gradients to cell centers.
for (int lev = base_level; lev <= finest_level; lev++)
{
BoxLib::average_face_to_cellcenter(grad_phi[lev], grad_phi_edge[lev], geom[lev]);
geom[lev].FillPeriodicBoundary(grad_phi[lev],true); // wz: why only fill periodic boundary?
}
// VisMF::Write(grad_phi,"GradPhi");
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real end = ParallelDescriptor::second() - strt;
#if 0
#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(end,IOProc);
if (ParallelDescriptor::IOProcessor())
std::cout << "solve_for_phi() time = " << end << std::endl;
#ifdef BL_LAZY
});
#endif
#endif
}
}