void setup_rhs(MultiFab& rhs, const Geometry& geom, Real a, Real b)
{
BL_PROFILE("setup_rhs");
Real sigma = 1.0;
Real w = 1.0;
int ibnd = static_cast<int>(bc_type);
// We test the sum of the RHS to check solvability
Real sum_rhs = 0.;
for ( MFIter mfi(rhs); mfi.isValid(); ++mfi ) {
const int* rlo = rhs[mfi].loVect();
const int* rhi = rhs[mfi].hiVect();
const Box& bx = mfi.validbox();
FORT_SET_RHS(rhs[mfi].dataPtr(),ARLIM(rlo),ARLIM(rhi),
bx.loVect(),bx.hiVect(),dx, a, b, sigma, w, ibnd);
sum_rhs += rhs[mfi].sum(0,1);
}
for (int n=0; n < BL_SPACEDIM; n++) {
sum_rhs *= dx[n];
}
ParallelDescriptor::ReduceRealSum(sum_rhs, ParallelDescriptor::IOProcessorNumber());
if (ParallelDescriptor::IOProcessor()) {
std::cout << "Sum of RHS : " << sum_rhs << std::endl;
}
}
void solve(MultiFab& soln, const MultiFab& anaSoln,
Real a, Real b, MultiFab& alpha, MultiFab beta[],
MultiFab& rhs, const BoxArray& bs, const Geometry& geom,
solver_t solver)
{
BL_PROFILE("solve");
soln.setVal(0.0);
const Real run_strt = ParallelDescriptor::second();
BndryData bd(bs, 1, geom);
set_boundary(bd, rhs);
ABecLaplacian abec_operator(bd, dx);
abec_operator.setScalars(a, b);
abec_operator.setCoefficients(alpha, beta);
MultiGrid mg(abec_operator);
mg.setMaxIter(maxiter);
mg.setVerbose(verbose);
mg.setFixedIter(1);
mg.solve(soln, rhs, tolerance_rel, tolerance_abs);
Real run_time = ParallelDescriptor::second() - run_strt;
ParallelDescriptor::ReduceRealMax(run_time, ParallelDescriptor::IOProcessorNumber());
if (ParallelDescriptor::IOProcessor()) {
std::cout << "Run time : " << run_time << std::endl;
}
}
void solve_with_Cpp(MultiFab& soln, MultiFab& gphi, Real a, Real b, MultiFab& alpha,
PArray<MultiFab>& beta, MultiFab& rhs, const BoxArray& bs, const Geometry& geom)
{
BL_PROFILE("solve_with_Cpp()");
BndryData bd(bs, 1, geom);
set_boundary(bd, rhs, 0);
ABecLaplacian abec_operator(bd, dx);
abec_operator.setScalars(a, b);
abec_operator.setCoefficients(alpha, beta);
MultiGrid mg(abec_operator);
mg.setVerbose(verbose);
mg.solve(soln, rhs, tolerance_rel, tolerance_abs);
PArray<MultiFab> grad_phi(BL_SPACEDIM, PArrayManage);
for (int n = 0; n < BL_SPACEDIM; ++n)
grad_phi.set(n, new MultiFab(BoxArray(soln.boxArray()).surroundingNodes(n), 1, 0));
#if (BL_SPACEDIM == 2)
abec_operator.compFlux(grad_phi[0],grad_phi[1],soln);
#elif (BL_SPACEDIM == 3)
abec_operator.compFlux(grad_phi[0],grad_phi[1],grad_phi[2],soln);
#endif
// Average edge-centered gradients to cell centers.
BoxLib::average_face_to_cellcenter(gphi, grad_phi, geom);
}
void
MultiGrid::average (MultiFab& c,
const MultiFab& f)
{
BL_PROFILE("MultiGrid::average()");
//
// Use Fortran function to average down (restrict) f to c.
//
const bool tiling = true;
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter cmfi(c,tiling); cmfi.isValid(); ++cmfi)
{
BL_ASSERT(c.boxArray().get(cmfi.index()) == cmfi.validbox());
const int nc = c.nComp();
const Box& bx = cmfi.tilebox();
FArrayBox& cfab = c[cmfi];
const FArrayBox& ffab = f[cmfi];
FORT_AVERAGE(cfab.dataPtr(),
ARLIM(cfab.loVect()), ARLIM(cfab.hiVect()),
ffab.dataPtr(),
ARLIM(ffab.loVect()), ARLIM(ffab.hiVect()),
bx.loVect(), bx.hiVect(), &nc);
}
}
void
MultiGrid::interpolate (MultiFab& f,
const MultiFab& c)
{
BL_PROFILE("MultiGrid::interpolate()");
//
// Use fortran function to interpolate up (prolong) c to f
// Note: returns f=f+P(c) , i.e. ADDS interp'd c to f.
//
// OMP over boxes
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter mfi(c); mfi.isValid(); ++mfi)
{
const int k = mfi.index();
const Box& bx = c.boxArray()[k];
const int nc = f.nComp();
const FArrayBox& cfab = c[mfi];
FArrayBox& ffab = f[mfi];
FORT_INTERP(ffab.dataPtr(),
ARLIM(ffab.loVect()), ARLIM(ffab.hiVect()),
cfab.dataPtr(),
ARLIM(cfab.loVect()), ARLIM(cfab.hiVect()),
bx.loVect(), bx.hiVect(), &nc);
}
}
void setup_coeffs(BoxArray& bs, MultiFab& alpha, PArray<MultiFab>& beta,
const Geometry& geom, MultiFab& cc_coef)
{
BL_PROFILE("setup_coeffs()");
ParmParse pp;
Real sigma, w;
pp.get("sigma", sigma);
pp.get("w", w);
for ( MFIter mfi(alpha); mfi.isValid(); ++mfi ) {
const Box& bx = mfi.validbox();
const int* alo = alpha[mfi].loVect();
const int* ahi = alpha[mfi].hiVect();
FORT_SET_ALPHA(alpha[mfi].dataPtr(),ARLIM(alo),ARLIM(ahi),
bx.loVect(),bx.hiVect(),dx);
const int* clo = cc_coef[mfi].loVect();
const int* chi = cc_coef[mfi].hiVect();
FORT_SET_CC_COEF(cc_coef[mfi].dataPtr(),ARLIM(clo),ARLIM(chi),
bx.loVect(),bx.hiVect(),dx, sigma, w);
}
BoxLib::average_cellcenter_to_face(beta, cc_coef, geom);
if (plot_beta == 1) {
writePlotFile("BETA", cc_coef, geom);
}
}
Real
ABecLaplacian::norm (int nm, int level, const bool local)
{
BL_PROFILE("ABecLaplacian::norm()");
BL_ASSERT(nm == 0);
const MultiFab& a = aCoefficients(level);
D_TERM(const MultiFab& bX = bCoefficients(0,level);,
void
BoxLib::WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels,
const Array<const MultiFab*>& mf,
const Array<std::string>& varnames,
const Array<Geometry>& geom, Real time, const Array<int>& level_steps,
const Array<IntVect>& ref_ratio)
{
BL_PROFILE("WriteMultiLevelPlotfile()");
BL_ASSERT(nlevels <= mf.size());
BL_ASSERT(nlevels <= geom.size());
BL_ASSERT(nlevels <= ref_ratio.size()+1);
BL_ASSERT(nlevels <= level_steps.size());
BL_ASSERT(mf[0]->nComp() == varnames.size());
int finest_level = nlevels-1;
//
// Only let 64 CPUs be writing at any one time.
//
int saveNFiles(VisMF::GetNOutFiles());
VisMF::SetNOutFiles(64);
const std::string versionName("HyperCLaw-V1.1");
const std::string levelPrefix("Level_");
const std::string mfPrefix("Cell");
bool callBarrier(true);
BoxLib::PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
if (ParallelDescriptor::IOProcessor()) {
std::string HeaderFileName(plotfilename + "/Header");
std::ofstream HeaderFile(HeaderFileName.c_str(), std::ofstream::out |
std::ofstream::trunc |
std::ofstream::binary);
if( ! HeaderFile.good()) {
BoxLib::FileOpenFailed(HeaderFileName);
}
Array<BoxArray> boxArrays(nlevels);
for(int level(0); level < boxArrays.size(); ++level) {
boxArrays[level] = mf[level]->boxArray();
}
BoxLib::WriteGenericPlotfileHeader(HeaderFile, nlevels, boxArrays, varnames,
geom, time, level_steps, ref_ratio);
}
for (int level = 0; level <= finest_level; ++level)
{
VisMF::Write(*mf[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
}
VisMF::SetNOutFiles(saveNFiles);
}
void set_boundary(BndryData& bd, const MultiFab& rhs, int comp=0)
{
BL_PROFILE("set_boundary");
Real bc_value = 0.0;
for (int n=0; n<BL_SPACEDIM; ++n) {
for (MFIter mfi(rhs); mfi.isValid(); ++mfi ) {
int i = mfi.index();
const Box& bx = mfi.validbox();
// Our default will be that the face of this grid is either touching another grid
// across an interior boundary or a periodic boundary. We will test for the other
// cases below.
{
// Define the type of boundary conditions to be Dirichlet (even for periodic)
bd.setBoundCond(Orientation(n, Orientation::low) ,i,comp,LO_DIRICHLET);
bd.setBoundCond(Orientation(n, Orientation::high),i,comp,LO_DIRICHLET);
// Set the boundary conditions to the cell centers outside the domain
bd.setBoundLoc(Orientation(n, Orientation::low) ,i,0.5*dx[n]);
bd.setBoundLoc(Orientation(n, Orientation::high),i,0.5*dx[n]);
}
// Now test to see if we should override the above with Dirichlet or Neumann physical bc's
if (bc_type != Periodic) {
int ibnd = static_cast<int>(bc_type); // either LO_DIRICHLET or LO_NEUMANN
const Geometry& geom = bd.getGeom();
// We are on the low side of the domain in coordinate direction n
if (bx.smallEnd(n) == geom.Domain().smallEnd(n)) {
// Set the boundary conditions to live exactly on the faces of the domain
bd.setBoundLoc(Orientation(n, Orientation::low) ,i,0.0 );
// Set the Dirichlet/Neumann boundary values
bd.setValue(Orientation(n, Orientation::low) ,i, bc_value);
// Define the type of boundary conditions
bd.setBoundCond(Orientation(n, Orientation::low) ,i,comp,ibnd);
}
// We are on the high side of the domain in coordinate direction n
if (bx.bigEnd(n) == geom.Domain().bigEnd(n)) {
// Set the boundary conditions to live exactly on the faces of the domain
bd.setBoundLoc(Orientation(n, Orientation::high) ,i,0.0 );
// Set the Dirichlet/Neumann boundary values
bd.setValue(Orientation(n, Orientation::high) ,i, bc_value);
// Define the type of boundary conditions
bd.setBoundCond(Orientation(n, Orientation::high) ,i,comp,ibnd);
}
}
}
}
}
void solve_with_hypre(MultiFab& soln, Real a, Real b, MultiFab& alpha,
PArray<MultiFab>& beta, MultiFab& rhs, const BoxArray& bs, const Geometry& geom)
{
BL_PROFILE("solve_with_hypre()");
BndryData bd(bs, 1, geom);
set_boundary(bd, rhs, 0);
HypreABecLap hypreSolver(bs, geom);
hypreSolver.setScalars(a, b);
hypreSolver.setACoeffs(alpha);
hypreSolver.setBCoeffs(beta);
hypreSolver.setVerbose(verbose);
hypreSolver.solve(soln, rhs, tolerance_rel, tolerance_abs, maxiter, bd);
}
void compute_analyticSolution(MultiFab& anaSoln, const Array<Real>& offset)
{
BL_PROFILE("compute_analyticSolution()");
int ibnd = static_cast<int>(bc_type);
for (MFIter mfi(anaSoln); mfi.isValid(); ++mfi) {
const int* alo = anaSoln[mfi].loVect();
const int* ahi = anaSoln[mfi].hiVect();
const Box& bx = mfi.validbox();
FORT_COMP_ASOL(anaSoln[mfi].dataPtr(), ARLIM(alo), ARLIM(ahi),
bx.loVect(),bx.hiVect(),dx, ibnd, offset.dataPtr());
}
}
void
CellBilinear::interp (const FArrayBox& crse,
int crse_comp,
FArrayBox& fine,
int fine_comp,
int ncomp,
const Box& fine_region,
const IntVect & ratio,
const Geometry& /*crse_geom*/,
const Geometry& /*fine_geom*/,
Array<BCRec>& /*bcr*/,
int actual_comp,
int actual_state)
{
BL_PROFILE("CellBilinear::interp()");
#if (BL_SPACEDIM == 3)
BoxLib::Error("interp: not implemented");
#endif
//
// Set up to call FORTRAN.
//
const int* clo = crse.box().loVect();
const int* chi = crse.box().hiVect();
const int* flo = fine.loVect();
const int* fhi = fine.hiVect();
const int* lo = fine_region.loVect();
const int* hi = fine_region.hiVect();
int num_slope = D_TERM(2,*2,*2)-1;
int len0 = crse.box().length(0);
int slp_len = num_slope*len0;
Array<Real> slope(slp_len);
int strp_len = len0*ratio[0];
Array<Real> strip(strp_len);
int strip_lo = ratio[0] * clo[0];
int strip_hi = ratio[0] * chi[0];
const Real* cdat = crse.dataPtr(crse_comp);
Real* fdat = fine.dataPtr(fine_comp);
const int* ratioV = ratio.getVect();
FORT_CBINTERP (cdat,ARLIM(clo),ARLIM(chi),ARLIM(clo),ARLIM(chi),
fdat,ARLIM(flo),ARLIM(fhi),ARLIM(lo),ARLIM(hi),
D_DECL(&ratioV[0],&ratioV[1],&ratioV[2]),&ncomp,
slope.dataPtr(),&num_slope,strip.dataPtr(),&strip_lo,&strip_hi,
&actual_comp,&actual_state);
}
void solve_with_F90(MultiFab& soln, MultiFab& gphi, Real a, Real b, MultiFab& alpha,
PArray<MultiFab>& beta, MultiFab& rhs, const BoxArray& bs, const Geometry& geom)
{
BL_PROFILE("solve_with_F90()");
FMultiGrid fmg(geom);
int mg_bc[2*BL_SPACEDIM];
if (bc_type == Periodic) {
// Define the type of boundary conditions to be periodic
for ( int n = 0; n < BL_SPACEDIM; ++n ) {
mg_bc[2*n + 0] = MGT_BC_PER;
mg_bc[2*n + 1] = MGT_BC_PER;
}
}
else if (bc_type == Neumann) {
// Define the type of boundary conditions to be Neumann
for ( int n = 0; n < BL_SPACEDIM; ++n ) {
mg_bc[2*n + 0] = MGT_BC_NEU;
mg_bc[2*n + 1] = MGT_BC_NEU;
}
}
else if (bc_type == Dirichlet) {
// Define the type of boundary conditions to be Dirichlet
for ( int n = 0; n < BL_SPACEDIM; ++n ) {
mg_bc[2*n + 0] = MGT_BC_DIR;
mg_bc[2*n + 1] = MGT_BC_DIR;
}
}
fmg.set_bc(mg_bc);
fmg.set_maxorder(maxorder);
fmg.set_scalars(a, b);
fmg.set_coefficients(alpha, beta);
int always_use_bnorm = 0;
int need_grad_phi = 1;
fmg.solve(soln, rhs, tolerance_rel, tolerance_abs, always_use_bnorm, need_grad_phi);
PArray<MultiFab> grad_phi(BL_SPACEDIM, PArrayManage);
for (int n = 0; n < BL_SPACEDIM; ++n)
grad_phi.set(n, new MultiFab(BoxArray(soln.boxArray()).surroundingNodes(n), 1, 0));
fmg.get_fluxes(grad_phi);
// Average edge-centered gradients to cell centers.
BoxLib::average_face_to_cellcenter(gphi, grad_phi, geom);
}
static
void
sxay (MultiFab& ss,
const MultiFab& xx,
Real a,
const MultiFab& yy,
int yycomp)
{
BL_PROFILE("CGSolver::sxay()");
const int ncomp = 1;
const int sscomp = 0;
const int xxcomp = 0;
MultiFab::LinComb(ss, 1.0, xx, xxcomp, a, yy, yycomp, sscomp, ncomp, 0);
}
//
// Do a one-component dot product of r & z using supplied components.
//
static
Real
dotxy (const MultiFab& r,
int rcomp,
const MultiFab& z,
int zcomp,
bool local)
{
BL_PROFILE("CGSolver::dotxy()");
BL_ASSERT(r.nComp() > rcomp);
BL_ASSERT(z.nComp() > zcomp);
BL_ASSERT(r.boxArray() == z.boxArray());
const int ncomp = 1;
const int nghost = 0;
return MultiFab::Dot(r,rcomp,z,zcomp,ncomp,nghost,local);
}
void
LinOp::residual (MultiFab& residL,
const MultiFab& rhsL,
MultiFab& solnL,
int level,
LinOp::BC_Mode bc_mode,
bool local)
{
BL_PROFILE("LinOp::residual()");
apply(residL, solnL, level, bc_mode, local);
const bool tiling = true;
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter solnLmfi(solnL,tiling); solnLmfi.isValid(); ++solnLmfi)
{
const int nc = residL.nComp();
//
// Only single-component solves supported (verified) by this class.
//
BL_ASSERT(nc == 1);
const Box& tbx = solnLmfi.tilebox();
BL_ASSERT(gbox[level][solnLmfi.index()] == solnLmfi.validbox());
FArrayBox& residfab = residL[solnLmfi];
const FArrayBox& rhsfab = rhsL[solnLmfi];
FORT_RESIDL(
residfab.dataPtr(),
ARLIM(residfab.loVect()), ARLIM(residfab.hiVect()),
rhsfab.dataPtr(),
ARLIM(rhsfab.loVect()), ARLIM(rhsfab.hiVect()),
residfab.dataPtr(),
ARLIM(residfab.loVect()), ARLIM(residfab.hiVect()),
tbx.loVect(), tbx.hiVect(), &nc);
}
}
void setup_rhs(MultiFab& rhs, const Geometry& geom)
{
BL_PROFILE("setup_rhs()");
ParmParse pp;
Real a, b, sigma, w;
pp.get("a", a);
pp.get("b", b);
pp.get("sigma", sigma);
pp.get("w", w);
int ibnd = static_cast<int>(bc_type);
// We test the sum of the RHS to check solvability
Real sum_rhs = 0.;
for ( MFIter mfi(rhs); mfi.isValid(); ++mfi ) {
const int* rlo = rhs[mfi].loVect();
const int* rhi = rhs[mfi].hiVect();
const Box& bx = rhs[mfi].box();
FORT_SET_RHS(rhs[mfi].dataPtr(),ARLIM(rlo),ARLIM(rhi),
bx.loVect(),bx.hiVect(),dx, a, b, sigma, w, ibnd);
sum_rhs += rhs[mfi].sum(0,1);
}
for (int n=0; n < BL_SPACEDIM; n++) {
sum_rhs *= dx[n];
}
ParallelDescriptor::ReduceRealSum(sum_rhs, ParallelDescriptor::IOProcessorNumber());
if (ParallelDescriptor::IOProcessor()) {
std::cout << "Sum of RHS : " << sum_rhs << std::endl;
}
if (plot_rhs == 1) {
writePlotFile("RHS", rhs, geom);
}
}
void
StateData::define (const Box& p_domain,
const BoxArray& grds,
const DistributionMapping& dm,
const StateDescriptor& d,
Real time,
Real dt)
{
BL_PROFILE("StateData::define()");
domain = p_domain;
desc = &d;
grids.define(grds);
//
// Convert to proper type.
//
IndexType typ(desc->getType());
StateDescriptor::TimeCenter t_typ(desc->timeType());
if (!typ.cellCentered())
{
domain.convert(typ);
grids.convert(typ);
}
if (t_typ == StateDescriptor::Point)
{
new_time.start = new_time.stop = time;
old_time.start = old_time.stop = time - dt;
}
else
{
new_time.start = time;
new_time.stop = time+dt;
old_time.start = time-dt;
old_time.stop = time;
}
int ncomp = desc->nComp();
new_data = new MultiFab(grids,ncomp,desc->nExtra(),dm,Fab_allocate);
old_data = 0;
}
void setup_coeffs(BoxArray& bs, MultiFab& alpha, MultiFab beta[], const Geometry& geom)
{
BL_PROFILE("setup_coeffs");
Real sigma = 1.0;
Real w = 1.0;
MultiFab cc_coef(bs,Ncomp,1); // cell-centered beta
for ( MFIter mfi(alpha); mfi.isValid(); ++mfi ) {
const Box& bx = mfi.validbox();
const int* alo = alpha[mfi].loVect();
const int* ahi = alpha[mfi].hiVect();
FORT_SET_ALPHA(alpha[mfi].dataPtr(),ARLIM(alo),ARLIM(ahi),
bx.loVect(),bx.hiVect(),dx);
const int* clo = cc_coef[mfi].loVect();
const int* chi = cc_coef[mfi].hiVect();
FORT_SET_CC_COEF(cc_coef[mfi].dataPtr(),ARLIM(clo),ARLIM(chi),
bx.loVect(),bx.hiVect(),dx, sigma, w);
}
// convert cell-centered beta to edges
for ( int n=0; n<BL_SPACEDIM; ++n ) {
for ( MFIter mfi(beta[n]); mfi.isValid(); ++mfi ) {
int i = mfi.index();
Box bx(bs[i]);
const int* clo = cc_coef[mfi].loVect();
const int* chi = cc_coef[mfi].hiVect();
const int* edgelo = beta[n][mfi].loVect();
const int* edgehi = beta[n][mfi].hiVect();
FORT_COEF_TO_EDGES(&n,beta[n][mfi].dataPtr(),ARLIM(edgelo),ARLIM(edgehi),
cc_coef[mfi].dataPtr(),ARLIM(clo),ARLIM(chi),
bx.loVect(),bx.hiVect());
}
}
}
void
NodeBilinear::interp (const FArrayBox& crse,
int crse_comp,
FArrayBox& fine,
int fine_comp,
int ncomp,
const Box& fine_region,
const IntVect& ratio,
const Geometry& /*crse_geom */,
const Geometry& /*fine_geom */,
Array<BCRec>& /*bcr*/,
int actual_comp,
int actual_state)
{
BL_PROFILE("NodeBilinear::interp()");
//
// Set up to call FORTRAN.
//
const int* clo = crse.box().loVect();
const int* chi = crse.box().hiVect();
const int* flo = fine.loVect();
const int* fhi = fine.hiVect();
const int* lo = fine_region.loVect();
const int* hi = fine_region.hiVect();
int num_slope = D_TERM(2,*2,*2)-1;
int len0 = crse.box().length(0);
int slp_len = num_slope*len0;
Array<Real> strip(slp_len);
const Real* cdat = crse.dataPtr(crse_comp);
Real* fdat = fine.dataPtr(fine_comp);
const int* ratioV = ratio.getVect();
FORT_NBINTERP (cdat,ARLIM(clo),ARLIM(chi),ARLIM(clo),ARLIM(chi),
fdat,ARLIM(flo),ARLIM(fhi),ARLIM(lo),ARLIM(hi),
D_DECL(&ratioV[0],&ratioV[1],&ratioV[2]),&ncomp,
strip.dataPtr(),&num_slope,&actual_comp,&actual_state);
}
void
CellConservativeLinear::interp (const FArrayBox& crse,
int crse_comp,
FArrayBox& fine,
int fine_comp,
int ncomp,
const Box& fine_region,
const IntVect& ratio,
const Geometry& crse_geom,
const Geometry& fine_geom,
Array<BCRec>& bcr,
int actual_comp,
int actual_state)
{
BL_PROFILE("CellConservativeLinear::interp()");
BL_ASSERT(bcr.size() >= ncomp);
//
// Make box which is intersection of fine_region and domain of fine.
//
Box target_fine_region = fine_region & fine.box();
//
// crse_bx is coarsening of target_fine_region, grown by 1.
//
Box crse_bx = CoarseBox(target_fine_region,ratio);
//
// Slopes are needed only on coarsening of target_fine_region.
//
Box cslope_bx(crse_bx);
cslope_bx.grow(-1);
//
// Make a refinement of cslope_bx
//
Box fine_version_of_cslope_bx = BoxLib::refine(cslope_bx,ratio);
//
// Get coarse and fine edge-centered volume coordinates.
//
Array<Real> fvc[BL_SPACEDIM];
Array<Real> cvc[BL_SPACEDIM];
int dir;
for (dir = 0; dir < BL_SPACEDIM; dir++)
{
fine_geom.GetEdgeVolCoord(fvc[dir],fine_version_of_cslope_bx,dir);
crse_geom.GetEdgeVolCoord(cvc[dir],crse_bx,dir);
}
//
// alloc tmp space for slope calc.
//
// In ucc_slopes and lcc_slopes , there is a slight abuse of
// the number of compenents argument
// --> there is a slope for each component in each coordinate
// direction
//
FArrayBox ucc_slopes(cslope_bx,ncomp*BL_SPACEDIM);
FArrayBox lcc_slopes(cslope_bx,ncomp*BL_SPACEDIM);
FArrayBox slope_factors(cslope_bx,BL_SPACEDIM);
FArrayBox cmax(cslope_bx,ncomp);
FArrayBox cmin(cslope_bx,ncomp);
FArrayBox alpha(cslope_bx,ncomp);
Real* fdat = fine.dataPtr(fine_comp);
const Real* cdat = crse.dataPtr(crse_comp);
Real* ucc_xsldat = ucc_slopes.dataPtr(0);
Real* lcc_xsldat = lcc_slopes.dataPtr(0);
Real* xslfac_dat = slope_factors.dataPtr(0);
#if (BL_SPACEDIM>=2)
Real* ucc_ysldat = ucc_slopes.dataPtr(ncomp);
Real* lcc_ysldat = lcc_slopes.dataPtr(ncomp);
Real* yslfac_dat = slope_factors.dataPtr(1);
#endif
#if (BL_SPACEDIM==3)
Real* ucc_zsldat = ucc_slopes.dataPtr(2*ncomp);
Real* lcc_zsldat = lcc_slopes.dataPtr(2*ncomp);
Real* zslfac_dat = slope_factors.dataPtr(2);
#endif
const int* flo = fine.loVect();
const int* fhi = fine.hiVect();
const int* clo = crse.loVect();
const int* chi = crse.hiVect();
const int* fblo = target_fine_region.loVect();
const int* fbhi = target_fine_region.hiVect();
const int* csbhi = cslope_bx.hiVect();
const int* csblo = cslope_bx.loVect();
int lin_limit = (do_linear_limiting ? 1 : 0);
const int* cvcblo = crse_bx.loVect();
const int* fvcblo = fine_version_of_cslope_bx.loVect();
int slope_flag = 1;
int cvcbhi[BL_SPACEDIM];
int fvcbhi[BL_SPACEDIM];
for (dir=0; dir<BL_SPACEDIM; dir++)
{
cvcbhi[dir] = cvcblo[dir] + cvc[dir].size() - 1;
fvcbhi[dir] = fvcblo[dir] + fvc[dir].size() - 1;
}
D_TERM(Real* voffx = new Real[fvc[0].size()]; ,
void
MultiGrid::coarsestSmooth (MultiFab& solL,
MultiFab& rhsL,
int level,
Real eps_rel,
Real eps_abs,
LinOp::BC_Mode bc_mode,
int local_usecg,
Real& cg_time)
{
BL_PROFILE("MultiGrid::coarsestSmooth()");
prepareForLevel(level);
if ( local_usecg == 0 )
{
Real error0 = 0;
if ( verbose > 0 )
{
error0 = errorEstimate(level, bc_mode);
if ( ParallelDescriptor::IOProcessor() )
std::cout << " Bottom Smoother: Initial error (error0) = "
<< error0 << '\n';
}
for (int i = finalSmooth(); i > 0; i--)
{
Lp.smooth(solL, rhsL, level, bc_mode);
if ( verbose > 1 || (i == 1 && verbose) )
{
Real error = errorEstimate(level, bc_mode);
const Real rel_error = (error0 != 0) ? error/error0 : 0;
if ( ParallelDescriptor::IOProcessor() )
std::cout << " Bottom Smoother: Iteration "
<< i
<< " error/error0 = "
<< rel_error << '\n';
}
}
}
else
{
bool use_mg_precond = false;
CGSolver cg(Lp, use_mg_precond, level);
cg.setMaxIter(maxiter_b);
const Real stime = ParallelDescriptor::second();
int ret = cg.solve(solL, rhsL, rtol_b, atol_b, bc_mode);
//
// The whole purpose of cg_time is to accumulate time spent in CGSolver.
//
cg_time += (ParallelDescriptor::second() - stime);
if ( ret != 0 )
{
if ( smooth_on_cg_unstable )
{
//
// If the CG solver returns a nonzero value indicating
// the problem is unstable. Assume this is not an accuracy
// issue and pound on it with the smoother.
// if ret == 8, then you have failure to converge
//
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
std::cout << "MultiGrid::coarsestSmooth(): CGSolver returns nonzero. Smoothing ...\n";
coarsestSmooth(solL, rhsL, level, eps_rel, eps_abs, bc_mode, 0, cg_time);
}
else
{
//
// cg failure probably indicates loss of precision accident.
// if ret == 8, then you have failure to converge
// setting solL to 0 should be ok.
//
solL.setVal(0);
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
std::cout << "MultiGrid::coarsestSmooth(): setting coarse corr to zero" << '\n';
}
}
}
for (int i = 0; i < nu_b; i++)
{
Lp.smooth(solL, rhsL, level, bc_mode);
}
}
}
void
MultiGrid::relax (MultiFab& solL,
MultiFab& rhsL,
int level,
Real eps_rel,
Real eps_abs,
LinOp::BC_Mode bc_mode,
Real& cg_time)
{
BL_PROFILE("MultiGrid::relax()");
//
// Recursively relax system. Equivalent to multigrid V-cycle.
// At coarsest grid, call coarsestSmooth.
//
if ( level < numlevels - 1 )
{
if ( verbose > 2 )
{
Real rnorm = errorEstimate(level, bc_mode);
if (ParallelDescriptor::IOProcessor())
{
std::cout << " AT LEVEL " << level << '\n';
std::cout << " DN:Norm before smooth " << rnorm << '\n';;
}
}
for (int i = preSmooth() ; i > 0 ; i--)
{
Lp.smooth(solL, rhsL, level, bc_mode);
}
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
if ( verbose > 2 )
{
Real rnorm = norm_inf(*res[level]);
if (ParallelDescriptor::IOProcessor())
std::cout << " DN:Norm after smooth " << rnorm << '\n';
}
prepareForLevel(level+1);
average(*rhs[level+1], *res[level]);
cor[level+1]->setVal(0.0);
for (int i = cntRelax(); i > 0 ; i--)
{
relax(*cor[level+1],*rhs[level+1],level+1,eps_rel,eps_abs,bc_mode,cg_time);
}
interpolate(solL, *cor[level+1]);
if ( verbose > 2 )
{
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
Real rnorm = norm_inf(*res[level]);
if ( ParallelDescriptor::IOProcessor() )
{
std::cout << " AT LEVEL " << level << '\n';
std::cout << " UP:Norm before smooth " << rnorm << '\n';
}
}
for (int i = postSmooth(); i > 0 ; i--)
{
Lp.smooth(solL, rhsL, level, bc_mode);
}
if ( verbose > 2 )
{
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
Real rnorm = norm_inf(*res[level]);
if ( ParallelDescriptor::IOProcessor() )
std::cout << " UP:Norm after smooth " << rnorm << '\n';
}
}
else
{
if ( verbose > 2 )
{
Real rnorm = norm_inf(rhsL);
if ( ParallelDescriptor::IOProcessor() )
{
std::cout << " AT LEVEL " << level << '\n';
std::cout << " DN:Norm before bottom " << rnorm << '\n';
}
}
coarsestSmooth(solL, rhsL, level, eps_rel, eps_abs, bc_mode, usecg, cg_time);
if ( verbose > 2 )
{
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
Real rnorm = norm_inf(*res[level]);
if ( ParallelDescriptor::IOProcessor() )
std::cout << " UP:Norm after bottom " << rnorm << '\n';
}
}
}
int
MultiGrid::solve_ (MultiFab& _sol,
Real eps_rel,
Real eps_abs,
LinOp::BC_Mode bc_mode,
Real bnorm,
Real resnorm0)
{
BL_PROFILE("MultiGrid::solve_()");
//
// If do_fixed_number_of_iters = 1, then do maxiter iterations without checking for convergence
//
// If do_fixed_number_of_iters = 0, then relax system maxiter times,
// and stop if relative error <= _eps_rel or if absolute err <= _abs_eps
//
const Real strt_time = ParallelDescriptor::second();
const int level = 0;
//
// We take the max of the norms of the initial RHS and the initial residual in order to capture both cases
//
Real norm_to_test_against;
bool using_bnorm;
if (bnorm >= resnorm0)
{
norm_to_test_against = bnorm;
using_bnorm = true;
} else {
norm_to_test_against = resnorm0;
using_bnorm = false;
}
int returnVal = 0;
Real error = resnorm0;
//
// Note: if eps_rel, eps_abs < 0 then that test is effectively bypassed
//
if ( ParallelDescriptor::IOProcessor() && eps_rel < 1.0e-16 && eps_rel > 0 )
{
std::cout << "MultiGrid: Tolerance "
<< eps_rel
<< " < 1e-16 is probably set too low" << '\n';
}
//
// We initially define norm_cor based on the initial solution only so we can use it in the very first iteration
// to decide whether the problem is already solved (this is relevant if the previous solve used was only solved
// according to the Anorm test and not the bnorm test).
//
Real norm_cor = norm_inf(*initialsolution,true);
ParallelDescriptor::ReduceRealMax(norm_cor);
int nit = 1;
const Real norm_Lp = Lp.norm(0, level);
Real cg_time = 0;
if ( use_Anorm_for_convergence == 1 )
{
//
// Don't need to go any further -- no iterations are required
//
if (error <= eps_abs || error < eps_rel*(norm_Lp*norm_cor+norm_to_test_against))
{
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
std::cout << " Problem is already converged -- no iterations required\n";
}
return 1;
}
for ( ;
( (error > eps_abs &&
error > eps_rel*(norm_Lp*norm_cor+norm_to_test_against)) ||
(do_fixed_number_of_iters == 1) )
&& nit <= maxiter;
++nit)
{
relax(*cor[level], *rhs[level], level, eps_rel, eps_abs, bc_mode, cg_time);
Real tmp[2] = { norm_inf(*cor[level],true), errorEstimate(level,bc_mode,true) };
ParallelDescriptor::ReduceRealMax(tmp,2);
norm_cor = tmp[0];
error = tmp[1];
if ( ParallelDescriptor::IOProcessor() && verbose > 1 )
{
const Real rel_error = error / norm_to_test_against;
Spacer(std::cout, level);
if (using_bnorm)
{
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/bnorm = "
<< rel_error << '\n';
} else {
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/resid0 = "
<< rel_error << '\n';
}
}
}
}
else
{
//
// Don't need to go any further -- no iterations are required
//
if (error <= eps_abs || error < eps_rel*norm_to_test_against)
{
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
std::cout << " Problem is already converged -- no iterations required\n";
}
return 1;
}
for ( ;
( (error > eps_abs &&
error > eps_rel*norm_to_test_against) ||
(do_fixed_number_of_iters == 1) )
&& nit <= maxiter;
++nit)
{
relax(*cor[level], *rhs[level], level, eps_rel, eps_abs, bc_mode, cg_time);
error = errorEstimate(level, bc_mode);
if ( ParallelDescriptor::IOProcessor() && verbose > 1 )
{
const Real rel_error = error / norm_to_test_against;
Spacer(std::cout, level);
if (using_bnorm)
{
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/bnorm = "
<< rel_error << '\n';
} else {
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/resid0 = "
<< rel_error << '\n';
}
}
}
}
Real run_time = (ParallelDescriptor::second() - strt_time);
if ( verbose > 0 )
{
if ( ParallelDescriptor::IOProcessor() )
{
const Real rel_error = error / norm_to_test_against;
Spacer(std::cout, level);
if (using_bnorm)
{
std::cout << "MultiGrid: Iteration "
<< nit-1
<< " resid/bnorm = "
<< rel_error << '\n';
} else {
std::cout << "MultiGrid: Iteration "
<< nit-1
<< " resid/resid0 = "
<< rel_error << '\n';
}
}
if ( verbose > 1 )
{
Real tmp[2] = { run_time, cg_time };
ParallelDescriptor::ReduceRealMax(tmp,2,ParallelDescriptor::IOProcessorNumber());
if ( ParallelDescriptor::IOProcessor() )
std::cout << ", Solve time: " << tmp[0] << ", CG time: " << tmp[1];
}
if ( ParallelDescriptor::IOProcessor() ) std::cout << '\n';
}
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
if ( do_fixed_number_of_iters == 1)
{
std::cout << " Did fixed number of iterations: " << maxiter << std::endl;
}
else if ( error < eps_rel*norm_to_test_against )
{
std::cout << " Converged res < eps_rel*max(bnorm,res_norm)\n";
}
else if ( (use_Anorm_for_convergence == 1) && (error < eps_rel*norm_Lp*norm_cor) )
{
std::cout << " Converged res < eps_rel*Anorm*sol\n";
}
else if ( error < eps_abs )
{
std::cout << " Converged res < eps_abs\n";
}
}
//
// Omit ghost update since maybe not initialized in calling routine.
// Add to boundary values stored in initialsolution.
//
_sol.copy(*cor[level]);
_sol.plus(*initialsolution,0,_sol.nComp(),0);
if ( use_Anorm_for_convergence == 1 )
{
if ( do_fixed_number_of_iters == 1 ||
error <= eps_rel*(norm_Lp*norm_cor+norm_to_test_against) ||
error <= eps_abs )
returnVal = 1;
}
else
{
if ( do_fixed_number_of_iters == 1 ||
error <= eps_rel*(norm_to_test_against) ||
error <= eps_abs )
returnVal = 1;
}
//
// Otherwise, failed to solve satisfactorily
//
return returnVal;
}
void
BndryRegister::defineDoit (Orientation _face,
IndexType _typ,
int _in_rad,
int _out_rad,
int _extent_rad,
BoxArray& fsBA)
{
BL_PROFILE("BndryRegister::defineDoit()");
BL_ASSERT(grids.size() > 0);
const int coord_dir = _face.coordDir();
const int lo_side = _face.isLow();
//
// Build the BoxArray on which to define the FabSet on this face.
//
const int N = grids.size();
fsBA.resize(N);
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (int idx = 0; idx < N; ++idx)
{
Box b;
//
// First construct proper box for direction normal to face.
//
if (_out_rad > 0)
{
if (_typ.ixType(coord_dir) == IndexType::CELL)
b = BoxLib::adjCell(grids[idx], _face, _out_rad);
else
b = BoxLib::bdryNode(grids[idx], _face, _out_rad);
if (_in_rad > 0)
b.grow(_face.flip(), _in_rad);
}
else
{
if (_in_rad > 0)
{
if (_typ.ixType(coord_dir) == IndexType::CELL)
b = BoxLib::adjCell(grids[idx], _face, _in_rad);
else
b = BoxLib::bdryNode(grids[idx], _face, _in_rad);
b.shift(coord_dir, lo_side ? _in_rad : -_in_rad);
}
else
BoxLib::Error("BndryRegister::define(): strange values for in_rad, out_rad");
}
//
// Now alter box in all other index directions.
//
for (int dir = 0; dir < BL_SPACEDIM; dir++)
{
if (dir == coord_dir)
continue;
if (_typ.ixType(dir) == IndexType::NODE)
b.surroundingNodes(dir);
if (_extent_rad > 0)
b.grow(dir,_extent_rad);
}
BL_ASSERT(b.ok());
fsBA.set(idx,b);
}
BL_ASSERT(fsBA.ok());
}
void
StateData::restart (std::istream& is,
const StateDescriptor& d,
const std::string& chkfile,
bool bReadSpecial)
{
BL_PROFILE("StateData::restart()");
if (bReadSpecial)
BoxLib::Abort("StateData:: restart:: w/bReadSpecial not implemented");
desc = &d;
is >> domain;
if (bReadSpecial)
{
BoxLib::readBoxArray(grids, is, bReadSpecial);
}
else
{
grids.readFrom(is);
}
is >> old_time.start;
is >> old_time.stop;
is >> new_time.start;
is >> new_time.stop;
int nsets;
is >> nsets;
old_data = 0;
new_data = 0;
//
// If no data is written then we just allocate the MF instead of reading it in.
// This assumes that the application will do something with it.
// We set it to zero in case a compiler complains about uninitialized data.
//
if (nsets == 0)
{
new_data = new MultiFab(grids,desc->nComp(),desc->nExtra(),Fab_allocate);
new_data->setVal(0.);
}
std::string mf_name;
std::string FullPathName;
//
// This reads the "new" data, if it's there.
//
if (nsets >= 1)
{
new_data = new MultiFab;
is >> mf_name;
//
// Note that mf_name is relative to the Header file.
// We need to prepend the name of the chkfile directory.
//
FullPathName = chkfile;
if (!chkfile.empty() && chkfile[chkfile.length()-1] != '/')
FullPathName += '/';
FullPathName += mf_name;
VisMF::Read(*new_data, FullPathName);
}
void
LinOp::applyBC (MultiFab& inout,
int src_comp,
int num_comp,
int level,
LinOp::BC_Mode bc_mode,
bool local,
int bndry_comp)
{
BL_PROFILE("LinOp::applyBC()");
//
// The inout MultiFab needs at least LinOp_grow ghost cells for applyBC.
//
BL_ASSERT(inout.nGrow() >= LinOp_grow);
//
// No coarsened boundary values, cannot apply inhomog at lev>0.
//
BL_ASSERT(level < numLevels());
BL_ASSERT(!(level > 0 && bc_mode == Inhomogeneous_BC));
int flagden = 1; // Fill in undrrelxr.
int flagbc = 1; // Fill boundary data.
if (bc_mode == LinOp::Homogeneous_BC)
//
// No data if homogeneous.
//
flagbc = 0;
const bool cross = true;
inout.FillBoundary(src_comp,num_comp,local,cross);
prepareForLevel(level);
//
// Do periodic fixup.
//
BL_ASSERT(level<geomarray.size());
geomarray[level].FillPeriodicBoundary(inout,src_comp,num_comp,false,local);
//
// Fill boundary cells.
//
// OMP over boxes
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter mfi(inout); mfi.isValid(); ++mfi)
{
const int gn = mfi.index();
BL_ASSERT(gbox[level][gn] == inout.box(gn));
BL_ASSERT(level<maskvals.size() && maskvals[level].local_index(gn)>=0);
BL_ASSERT(level<lmaskvals.size() && lmaskvals[level].local_index(gn)>=0);
BL_ASSERT(level<undrrelxr.size());
const MaskTuple& ma = maskvals[level][gn];
const MaskTuple& lma = lmaskvals[level][gn];
const BndryData::RealTuple& bdl = bgb->bndryLocs(gn);
const Array< Array<BoundCond> >& bdc = bgb->bndryConds(gn);
for (OrientationIter oitr; oitr; ++oitr)
{
const Orientation o = oitr();
FabSet& f = (*undrrelxr[level])[o];
int cdr = o;
const FabSet& fs = bgb->bndryValues(o);
const Mask& m = local ? (*lma[o]) : (*ma[o]);
Real bcl = bdl[o];
BL_ASSERT(bdc[o].size()>bndry_comp);
int bct = bdc[o][bndry_comp];
const Box& vbx = inout.box(gn);
FArrayBox& iofab = inout[mfi];
BL_ASSERT(f.size()>gn);
BL_ASSERT(fs.size()>gn);
FArrayBox& ffab = f[mfi];
const FArrayBox& fsfab = fs[mfi];
FORT_APPLYBC(&flagden, &flagbc, &maxorder,
iofab.dataPtr(src_comp),
ARLIM(iofab.loVect()), ARLIM(iofab.hiVect()),
&cdr, &bct, &bcl,
fsfab.dataPtr(bndry_comp),
ARLIM(fsfab.loVect()), ARLIM(fsfab.hiVect()),
m.dataPtr(),
ARLIM(m.loVect()), ARLIM(m.hiVect()),
ffab.dataPtr(),
ARLIM(ffab.loVect()), ARLIM(ffab.hiVect()),
vbx.loVect(),
vbx.hiVect(), &num_comp, h[level]);
}
}
}
void
BoxLib::WriteGenericPlotfileHeader (std::ostream &HeaderFile,
int nlevels,
const Array<BoxArray> &bArray,
const Array<std::string> &varnames,
const Array<Geometry> &geom,
Real time,
const Array<int> &level_steps,
const Array<IntVect> &ref_ratio,
const std::string &versionName,
const std::string &levelPrefix,
const std::string &mfPrefix)
{
BL_PROFILE("WriteGenericPlotfileHeader()");
BL_ASSERT(nlevels <= bArray.size());
BL_ASSERT(nlevels <= geom.size());
BL_ASSERT(nlevels <= ref_ratio.size()+1);
BL_ASSERT(nlevels <= level_steps.size());
int finest_level(nlevels - 1);
HeaderFile.precision(17);
VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
// ---- this is the generic plot file type name
HeaderFile << versionName << '\n';
HeaderFile << varnames.size() << '\n';
for (int ivar = 0; ivar < varnames.size(); ++ivar) {
HeaderFile << varnames[ivar] << "\n";
}
HeaderFile << BL_SPACEDIM << '\n';
HeaderFile << time << '\n';
HeaderFile << finest_level << '\n';
for (int i = 0; i < BL_SPACEDIM; ++i) {
HeaderFile << Geometry::ProbLo(i) << ' ';
}
HeaderFile << '\n';
for (int i = 0; i < BL_SPACEDIM; ++i) {
HeaderFile << Geometry::ProbHi(i) << ' ';
}
HeaderFile << '\n';
for (int i = 0; i < finest_level; ++i) {
HeaderFile << ref_ratio[i][0] << ' ';
}
HeaderFile << '\n';
for (int i = 0; i <= finest_level; ++i) {
HeaderFile << geom[i].Domain() << ' ';
}
HeaderFile << '\n';
for (int i = 0; i <= finest_level; ++i) {
HeaderFile << level_steps[i] << ' ';
}
HeaderFile << '\n';
for (int i = 0; i <= finest_level; ++i) {
for (int k = 0; k < BL_SPACEDIM; ++k) {
HeaderFile << geom[i].CellSize()[k] << ' ';
}
HeaderFile << '\n';
}
HeaderFile << (int) Geometry::Coord() << '\n';
HeaderFile << "0\n";
for (int level = 0; level <= finest_level; ++level) {
HeaderFile << level << ' ' << bArray[level].size() << ' ' << time << '\n';
HeaderFile << level_steps[level] << '\n';
for (int i = 0; i < bArray[level].size(); ++i)
{
const Box &b(bArray[level][i]);
RealBox loc = RealBox(b, geom[level].CellSize(), geom[level].ProbLo());
for (int n = 0; n < BL_SPACEDIM; ++n) {
HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
}
}
HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
}
}
void
LinOp::prepareForLevel (int level)
{
BL_PROFILE("LinOp::prepareForLevel()");
if (level == 0) return;
LinOp::prepareForLevel(level-1);
if (h.size() > level) return;
//
// Assume from here down that this is a new level one coarser than existing
//
BL_ASSERT(h.size() == level);
h.resize(level+1);
for (int i = 0; i < BL_SPACEDIM; ++i)
{
h[level][i] = h[level-1][i]*2.0;
}
geomarray.resize(level+1);
geomarray[level].define(BoxLib::coarsen(geomarray[level-1].Domain(),2));
const Box& curdomain = geomarray[level].Domain();
//
// Add a box to the new coarser level (assign removes old BoxArray).
//
gbox.resize(level+1);
gbox[level] = gbox[level-1];
gbox[level].coarsen(2);
//
// Add the BndryRegister of relax values to the new coarser level.
//
BL_ASSERT(undrrelxr.size() == level);
undrrelxr.resize(level+1);
undrrelxr[level] = new BndryRegister(gbox[level], 1, 0, 0, 1);
//
// Add an Array of Array of maskvals to the new coarser level
// For each orientation, build NULL masks, then use distributed allocation
// Initial masks for coarse levels, ignore outside_domain possibility since
// we always solve homogeneous equation on coarse levels.
//
BL_ASSERT( maskvals.size() == level);
BL_ASSERT(lmaskvals.size() == level);
maskvals.resize(level+1);
lmaskvals.resize(level+1);
Array<IntVect> pshifts(27);
std::vector< std::pair<int,Box> > isects;
//
// Use bgb's distribution map for masks.
// We note that all orientations of the FabSets have the same distribution.
// We'll use the low 0 side as the model.
//
maskvals[level].reserve((*bgb)[Orientation(0,Orientation::low)].local_size());
lmaskvals[level].reserve((*bgb)[Orientation(0,Orientation::low)].local_size());
for (FabSetIter bndryfsi((*bgb)[Orientation(0,Orientation::low)]);
bndryfsi.isValid();
++bndryfsi)
{
const int gn = bndryfsi.index();
MaskTuple& ma = maskvals[level][gn];
MaskTuple& lma = lmaskvals[level][gn];
for (OrientationIter oitr; oitr; ++oitr)
{
const Orientation face = oitr();
const Box& bx_k = BoxLib::adjCell(gbox[level][gn], face, 1);
ma[face] = new Mask(bx_k,1);
lma[face] = new Mask(bx_k,1);
Mask& curmask = *( ma[face]);
Mask& lcurmask = *(lma[face]);
curmask.setVal(BndryData::not_covered);
lcurmask.setVal(BndryData::not_covered);
gbox[level].intersections(bx_k,isects);
for (int ii = 0, N = isects.size(); ii < N; ii++)
{
if (isects[ii].first != gn)
{
curmask.setVal(BndryData::covered, isects[ii].second, 0);
lcurmask.setVal(BndryData::covered, isects[ii].second, 0);
}
}
//
// Now take care of periodic wraparounds.
//
Geometry& curgeom = geomarray[level];
if (curgeom.isAnyPeriodic() && !curdomain.contains(bx_k))
{
curgeom.periodicShift(curdomain, bx_k, pshifts);
for (int iiv = 0, M = pshifts.size(); iiv < M; iiv++)
{
curmask.shift(pshifts[iiv]);
lcurmask.shift(pshifts[iiv]);
BL_ASSERT(curmask.box() == lcurmask.box());
gbox[level].intersections(curmask.box(),isects);
for (int ii = 0, N = isects.size(); ii < N; ii++)
{
curmask.setVal(BndryData::covered, isects[ii].second, 0);
lcurmask.setVal(BndryData::covered, isects[ii].second, 0);
}
curmask.shift(-pshifts[iiv]);
lcurmask.shift(-pshifts[iiv]);
}
}
}
}
gbox[level].clear_hash_bin();
}
void
LinOp::makeCoefficients (MultiFab& cs,
const MultiFab& fn,
int level)
{
BL_PROFILE("LinOp::makeCoefficients()");
int nc = 1;
//
// Determine index type of incoming MultiFab.
//
const IndexType iType(fn.boxArray().ixType());
const IndexType cType(D_DECL(IndexType::CELL, IndexType::CELL, IndexType::CELL));
const IndexType xType(D_DECL(IndexType::NODE, IndexType::CELL, IndexType::CELL));
const IndexType yType(D_DECL(IndexType::CELL, IndexType::NODE, IndexType::CELL));
#if (BL_SPACEDIM == 3)
const IndexType zType(D_DECL(IndexType::CELL, IndexType::CELL, IndexType::NODE));
#endif
int cdir;
if (iType == cType)
{
cdir = -1;
}
else if (iType == xType)
{
cdir = 0;
}
else if (iType == yType)
{
cdir = 1;
#if (BL_SPACEDIM == 3)
}
else if (iType == zType)
{
cdir = 2;
#endif
}
else
{
BoxLib::Error("LinOp::makeCoeffients: Bad index type");
}
BoxArray d(gbox[level]);
if (cdir >= 0)
d.surroundingNodes(cdir);
//
// Only single-component solves supported (verified) by this class.
//
const int nComp=1;
const int nGrow=0;
cs.define(d, nComp, nGrow, Fab_allocate);
const bool tiling = true;
switch (cdir)
{
case -1:
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter csmfi(cs,tiling); csmfi.isValid(); ++csmfi)
{
const Box& tbx = csmfi.tilebox();
FArrayBox& csfab = cs[csmfi];
const FArrayBox& fnfab = fn[csmfi];
FORT_AVERAGECC(csfab.dataPtr(), ARLIM(csfab.loVect()),
ARLIM(csfab.hiVect()),fnfab.dataPtr(),
ARLIM(fnfab.loVect()),ARLIM(fnfab.hiVect()),
tbx.loVect(),tbx.hiVect(), &nc);
}
break;
case 0:
case 1:
case 2:
if (harmavg)
{
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter csmfi(cs,tiling); csmfi.isValid(); ++csmfi)
{
const Box& tbx = csmfi.tilebox();
FArrayBox& csfab = cs[csmfi];
const FArrayBox& fnfab = fn[csmfi];
FORT_HARMONIC_AVERAGEEC(csfab.dataPtr(),
ARLIM(csfab.loVect()),
ARLIM(csfab.hiVect()),
fnfab.dataPtr(),
ARLIM(fnfab.loVect()),
ARLIM(fnfab.hiVect()),
tbx.loVect(),tbx.hiVect(),
&nc,&cdir);
}
}
else
{
#ifdef _OPENMP
#pragma omp parallel
#endif
for (MFIter csmfi(cs,tiling); csmfi.isValid(); ++csmfi)
{
const Box& tbx = csmfi.tilebox();
FArrayBox& csfab = cs[csmfi];
const FArrayBox& fnfab = fn[csmfi];
FORT_AVERAGEEC(csfab.dataPtr(),ARLIM(csfab.loVect()),
ARLIM(csfab.hiVect()),fnfab.dataPtr(),
ARLIM(fnfab.loVect()),ARLIM(fnfab.hiVect()),
tbx.loVect(),tbx.hiVect(),
&nc, &cdir);
}
}
break;
default:
BoxLib::Error("LinOp:: bad coefficient coarsening direction!");
}
}