double
Multigrid::resnorm(
LevelData<double >& a_phi,
const LevelData<double >& a_rhs
)
{
CH_TIMERS("Multigrid::resnorm");
double retval = 0;
a_phi.exchange();
// BLIterator blit(m_bl);
for (BLIterator blit(m_bl); blit != blit.end(); ++blit)
// for (blit.begin();blit != blit.end();++blit)
{
Point pt = *blit;
RectMDArray<double > res(a_rhs[pt].getBox());
res.setVal(0.0);
res |= g_Laplacian(a_phi[*blit],res.getBox());
res /= m_dx*m_dx;
res -= a_rhs[*blit];
double resmax = abs_max(res,res.getBox());
if (retval < resmax) retval = resmax;
}
return retval;
};
void
Multigrid::pointRelax(
LevelData<double >& a_phi,
const LevelData<double >& a_rhs,
int a_numIter
)
{
CH_TIMERS("Multigrid::pointRelax");
CH_TIMER("sec1",t1);
CH_TIMER("sec2",t2);
CH_TIMER("sec3",t3);
BLIterator blit(m_bl);
for (int iter = 0; iter < a_numIter; iter++)
{
a_phi.exchange();
for (blit.begin();blit != blit.end();++blit)
{
RectMDArray<double > res(m_bl[*blit]);
RectMDArray<double > tmp(m_bl[*blit]);
CH_START(t1);
res |= g_Laplacian(a_phi[*blit],res.getBox());
res /= m_dx*m_dx;
CH_STOP(t1);
CH_START(t2);
res -= a_rhs[*blit];
CH_STOP(t2);
CH_START(t3);
tmp |= g_PointRelax(res,res.getBox());
tmp *= m_dx*m_dx;
a_phi[*blit] += tmp;
CH_STOP(t3);
}
}
};
// ---------------------------------------------------------
void
AMRNavierStokes::computeLapVel(LevelData<FArrayBox>& a_lapVel,
LevelData<FArrayBox>& a_vel,
const LevelData<FArrayBox>* a_crseVelPtr)
{
// set BC's
VelBCHolder velBC(m_physBCPtr->viscousVelFuncBC());
bool isHomogeneous = false;
m_velocityAMRPoissonOp.applyOp(a_lapVel, a_vel, a_crseVelPtr,
isHomogeneous, velBC);
// may need to extend lapVel to cover ghost cells as well
{
BCHolder viscBC = m_physBCPtr->viscousFuncBC();
const DisjointBoxLayout& grids = a_lapVel.getBoxes();
DataIterator dit = a_lapVel.dataIterator();
for (dit.reset(); dit.ok(); ++dit)
{
viscBC(a_lapVel[dit], grids[dit],
m_problem_domain, m_dx,
false); // not homogeneous
}
}
// finally, do exchange
a_lapVel.exchange(a_lapVel.interval());
}
// ---------------------------------------------------------
void
AMRNavierStokes::computeLapScal(LevelData<FArrayBox>& a_lapScal,
LevelData<FArrayBox>& a_scal,
const BCHolder& a_physBC,
const LevelData<FArrayBox>* a_crseScalPtr)
{
m_scalarsAMRPoissonOp.setBC(a_physBC);
bool isHomogeneous = false;
if (a_crseScalPtr != NULL)
{
m_scalarsAMRPoissonOp.AMROperatorNF(a_lapScal, a_scal, *a_crseScalPtr,
isHomogeneous);
}
else
{
m_scalarsAMRPoissonOp.applyOpI(a_lapScal, a_scal,
isHomogeneous);
}
BCHolder viscBC = m_physBCPtr->viscousFuncBC();
DataIterator dit = a_lapScal.dataIterator();
const DisjointBoxLayout& grids = a_lapScal.getBoxes();
for (dit.reset(); dit.ok(); ++dit)
{
viscBC(a_lapScal[dit],
grids[dit],
m_problem_domain, m_dx,
false); // not homogeneous
}
// finally, do exchange
a_lapScal.exchange(a_lapScal.interval());
}
void
EBLevelTransport::
fillCons(LevelData<EBCellFAB>& a_consState,
LevelData<EBCellFAB>& a_normalVel,
const LevelData<EBCellFAB>& a_consStateCoarseOld,
const LevelData<EBCellFAB>& a_consStateCoarseNew,
const LevelData<EBCellFAB>& a_normalVelCoarseOld,
const LevelData<EBCellFAB>& a_normalVelCoarseNew,
const Real& a_time,
const Real& a_coarTimeOld,
const Real& a_coarTimeNew)
{
Interval consInterv(0, m_nCons-1);
Interval fluxInterv(0, m_nFlux-1);
Interval intervSD(0,SpaceDim-1);
if (m_hasCoarser)
{
m_fillPatch.interpolate(a_consState,
a_consStateCoarseOld,
a_consStateCoarseNew,
a_coarTimeOld,
a_coarTimeNew,
a_time,
consInterv);
m_fillPatchVel.interpolate(a_normalVel,
a_normalVelCoarseOld,
a_normalVelCoarseNew,
a_coarTimeOld,
a_coarTimeNew,
a_time,
intervSD);
}
// Exchange all the data between grids
a_consState.exchange(consInterv);
a_normalVel.exchange(intervSD);
}
void
Multigrid::residual(
LevelData<double >& a_res,
LevelData<double >& a_phi,
const LevelData<double >& a_rhs
)
{
CH_TIMERS("Multigrid::residual");
a_phi.exchange();
for (BLIterator blit(m_bl); blit != blit.end(); ++blit)
{
Point pt = *blit;
RectMDArray<double >& res = a_res[pt];
res |= g_Laplacian(a_phi[*blit],res.getBox());
res /= m_dx*m_dx;
res *= -1.0;
res += a_rhs[*blit];
}
};
void EBPoissonOp::
colorGS(LevelData<EBCellFAB>& a_phi,
const LevelData<EBCellFAB>& a_rhs,
const IntVect& a_color,
int icolor)
{
CH_TIME("EBPoissonOp::colorGS");
//this is a multigrid operator so only homogeneous CF BC and null coar level
CH_assert(a_rhs.ghostVect() == m_ghostCellsRHS);
CH_assert(a_phi.ghostVect() == m_ghostCellsPhi);
a_phi.exchange();
Real weight = m_alpha;
for (int idir = 0; idir < SpaceDim; idir++)
{
weight += -2.0 * m_beta * m_invDx2[idir];
}
weight = 1.0 / weight;
for (DataIterator dit = a_phi.dataIterator(); dit.ok(); ++dit)
{
EBCellFAB& phifab = a_phi[dit()];
m_colorEBStencil[icolor][dit()]->cache(phifab);
}
GSColorAllRegular( a_phi, a_rhs, a_color, weight, true);
for (DataIterator dit = a_phi.dataIterator(); dit.ok(); ++dit)
{
EBCellFAB& phifab = a_phi[dit()];
m_colorEBStencil[icolor][dit()]->uncache(phifab);
}
GSColorAllIrregular(a_phi, a_rhs, a_color, true, icolor);
}
void testlap(LevelData<double >& a_phi, double a_dx,char* a_str)
{
double coef = 1./(a_dx*a_dx);
Stencil<double> Laplacian(make_pair(getZeros(),-DIM*2*coef));
BoxLayout bl = a_phi.getBoxLayout();
for (int dir = 0; dir < DIM ; dir++)
{
Point edir = getUnitv(dir);
Stencil<double> plus(make_pair(Shift(edir),coef));
Stencil<double> minus(make_pair(Shift(edir*(-1)),coef));
Laplacian = Laplacian + minus + plus;
}
a_phi.exchange();
RectMDArray<double> LPhi00(bl.getDomain());
for (BLIterator blit(bl); blit != blit.end(); ++blit)
{
LPhi00 |= Laplacian(a_phi[*blit],bl[*blit]);
}
MDWrite(a_str,LPhi00);
};
void
EBMGInterp::fillGhostCellsPWC(LevelData<EBCellFAB>& a_data,
const EBISLayout& a_ebisl,
const ProblemDomain& a_dom)
{
for (int idir = 0; idir < SpaceDim; idir++)
{
if (m_ghost[idir] < 1)
{
MayDay::Error("EBMGInterp:I need a ghost cell for linear interpolation");
}
}
//extrapolate to every ghost cell
const DisjointBoxLayout& dbl = a_data.disjointBoxLayout();
for (DataIterator dit = dbl.dataIterator(); dit.ok(); ++dit)
{
const Box& grid = dbl.get(dit());
const EBGraph& ebgraph = a_ebisl[dit()].getEBGraph();
for (int idir = 0; idir < SpaceDim; idir++)
{
for (SideIterator sit; sit.ok(); ++sit)
{
Side::LoHiSide flipSide = flip(sit());
Box ghostBox = adjCellBox(grid, idir, sit(), 1);
for (BoxIterator boxit(ghostBox); boxit.ok(); ++boxit)
{
if (a_dom.contains(boxit()))
{
Vector<VolIndex> vofs = ebgraph.getVoFs(boxit());
for (int ivof = 0; ivof < vofs.size(); ivof++)
{
Real extrapVal = 0;
Vector<FaceIndex> faces = ebgraph.getFaces(vofs[ivof], idir, flipSide);
for (int ivar = 0; ivar < a_data.nComp(); ivar++)
{
for (int iface = 0; iface < faces.size(); iface++)
{
const VolIndex& flipVoF = faces[iface].getVoF(flipSide);
extrapVal += a_data[dit()](flipVoF, ivar);
}
if (faces.size() > 1) extrapVal /= faces.size();
a_data[dit()](vofs[ivof], ivar) = extrapVal;
}
}
}
else
{
//just put in the single valued part of the data holder something sensible
//if not inside domain
int isign = sign(flipSide);
BaseFab<Real>& bfdata = a_data[dit()].getSingleValuedFAB();
IntVect otherIV = boxit() + isign*BASISV(idir);
for (int ivar = 0; ivar < a_data.nComp(); ivar++)
{
bfdata(boxit(), ivar) = bfdata(otherIV, ivar);
}
}
}
}
}
}
//do exchange to overwrite ghost cells where there exists real data
a_data.exchange();
}
void VCAMRPoissonOp2::looseGSRB(LevelData<FArrayBox>& a_phi,
const LevelData<FArrayBox>& a_rhs)
{
CH_TIME("VCAMRPoissonOp2::looseGSRB");
#if 1
MayDay::Abort("VCAMRPoissonOp2::looseGSRB - Not implemented");
#else
// This implementation converges at half the rate of "levelGSRB" in
// multigrid solves
CH_assert(a_phi.isDefined());
CH_assert(a_rhs.isDefined());
CH_assert(a_phi.ghostVect() >= IntVect::Unit);
CH_assert(a_phi.nComp() == a_rhs.nComp());
// Recompute the relaxation coefficient if needed.
resetLambda();
const DisjointBoxLayout& dbl = a_phi.disjointBoxLayout();
DataIterator dit = a_phi.dataIterator();
// fill in intersection of ghostcells and a_phi's boxes
{
CH_TIME("VCAMRPoissonOp2::looseGSRB::homogeneousCFInterp");
homogeneousCFInterp(a_phi);
}
{
CH_TIME("VCAMRPoissonOp2::looseGSRB::exchange");
a_phi.exchange(a_phi.interval(), m_exchangeCopier);
}
// now step through grids...
for (dit.begin(); dit.ok(); ++dit)
{
// invoke physical BC's where necessary
{
CH_TIME("VCAMRPoissonOp2::looseGSRB::BCs");
m_bc(a_phi[dit], dbl[dit()], m_domain, m_dx, true);
}
const Box& region = dbl.get(dit());
const FluxBox& thisBCoef = (*m_bCoef)[dit];
int whichPass = 0;
#if CH_SPACEDIM == 1
FORT_GSRBHELMHOLTZVC1D
#elif CH_SPACEDIM == 2
FORT_GSRBHELMHOLTZVC2D
#elif CH_SPACEDIM == 3
FORT_GSRBHELMHOLTZVC3D
#else
This_will_not_compile!
#endif
(CHF_FRA(a_phi[dit]),
CHF_CONST_FRA(a_rhs[dit]),
CHF_BOX(region),
CHF_CONST_REAL(m_dx),
CHF_CONST_REAL(m_alpha),
CHF_CONST_FRA((*m_aCoef)[dit]),
CHF_CONST_REAL(m_beta),
#if CH_SPACEDIM >= 1
CHF_CONST_FRA(thisBCoef[0]),
#endif
#if CH_SPACEDIM >= 2
CHF_CONST_FRA(thisBCoef[1]),
#endif
#if CH_SPACEDIM >= 3
CHF_CONST_FRA(thisBCoef[2]),
#endif
#if CH_SPACEDIM >= 4
This_will_not_compile!
#endif
CHF_CONST_FRA(m_lambda[dit]),
CHF_CONST_INT(whichPass));
whichPass = 1;
#if CH_SPACEDIM == 1
FORT_GSRBHELMHOLTZVC1D
#elif CH_SPACEDIM == 2
FORT_GSRBHELMHOLTZVC2D
#elif CH_SPACEDIM == 3
FORT_GSRBHELMHOLTZVC3D
#else
This_will_not_compile!
#endif
(CHF_FRA(a_phi[dit]),
CHF_CONST_FRA(a_rhs[dit]),
CHF_BOX(region),
CHF_CONST_REAL(m_dx),
CHF_CONST_REAL(m_alpha),
CHF_CONST_FRA((*m_aCoef)[dit]),
CHF_CONST_REAL(m_beta),
#if CH_SPACEDIM >= 1
CHF_CONST_FRA(thisBCoef[0]),
#endif
#if CH_SPACEDIM >= 2
CHF_CONST_FRA(thisBCoef[1]),
#endif
#if CH_SPACEDIM >= 3
CHF_CONST_FRA(thisBCoef[2]),
#endif
#if CH_SPACEDIM >= 4
This_will_not_compile!
#endif
CHF_CONST_FRA(m_lambda[dit]),
CHF_CONST_INT(whichPass));
} // end loop through grids
#endif
}
void VCAMRPoissonOp2::restrictResidual(LevelData<FArrayBox>& a_resCoarse,
LevelData<FArrayBox>& a_phiFine,
const LevelData<FArrayBox>& a_rhsFine)
{
CH_TIME("VCAMRPoissonOp2::restrictResidual");
homogeneousCFInterp(a_phiFine);
const DisjointBoxLayout& dblFine = a_phiFine.disjointBoxLayout();
for (DataIterator dit = a_phiFine.dataIterator(); dit.ok(); ++dit)
{
FArrayBox& phi = a_phiFine[dit];
m_bc(phi, dblFine[dit()], m_domain, m_dx, true);
}
a_phiFine.exchange(a_phiFine.interval(), m_exchangeCopier);
for (DataIterator dit = a_phiFine.dataIterator(); dit.ok(); ++dit)
{
FArrayBox& phi = a_phiFine[dit];
const FArrayBox& rhs = a_rhsFine[dit];
FArrayBox& res = a_resCoarse[dit];
const FArrayBox& thisACoef = (*m_aCoef)[dit];
const FluxBox& thisBCoef = (*m_bCoef)[dit];
Box region = dblFine.get(dit());
const IntVect& iv = region.smallEnd();
IntVect civ = coarsen(iv, 2);
res.setVal(0.0);
#if CH_SPACEDIM == 1
FORT_RESTRICTRESVC1D
#elif CH_SPACEDIM == 2
FORT_RESTRICTRESVC2D
#elif CH_SPACEDIM == 3
FORT_RESTRICTRESVC3D
#else
This_will_not_compile!
#endif
(CHF_FRA_SHIFT(res, civ),
CHF_CONST_FRA_SHIFT(phi, iv),
CHF_CONST_FRA_SHIFT(rhs, iv),
CHF_CONST_REAL(m_alpha),
CHF_CONST_FRA_SHIFT(thisACoef, iv),
CHF_CONST_REAL(m_beta),
#if CH_SPACEDIM >= 1
CHF_CONST_FRA_SHIFT(thisBCoef[0], iv),
#endif
#if CH_SPACEDIM >= 2
CHF_CONST_FRA_SHIFT(thisBCoef[1], iv),
#endif
#if CH_SPACEDIM >= 3
CHF_CONST_FRA_SHIFT(thisBCoef[2], iv),
#endif
#if CH_SPACEDIM >= 4
This_will_not_compile!
#endif
CHF_BOX_SHIFT(region, iv),
CHF_CONST_REAL(m_dx));
}
}
void
EBGradDivFilter::
filter(LevelData<EBCellFAB>& a_velFine,
const LevelData<EBFluxFAB>& a_fluxVelFine,
const LevelData<EBCellFAB>* a_velCoar,
bool a_lowOrderOneSidedGrad,
bool a_noExtrapToCovered)
{
CH_TIME("EBGradDivFilter::filter");
CH_assert(a_velFine.nComp() == SpaceDim);
Interval velInterv(0, SpaceDim-1);
//do coarse-fine interpolation if necessary
if (m_hasCoarser)
{
CH_assert(a_velCoar != NULL);
CH_assert(a_velCoar->nComp() == SpaceDim);
m_tensorCFI->coarseFineInterp(a_velFine, m_gradVel, *a_velCoar);
}
//fill in ghost cells
a_velFine.exchange(velInterv);
//compute gradient of velocity for parts NOT in ghost cells
for (DataIterator dit = m_gridsFine.dataIterator(); dit.ok(); ++dit)
{
gradVel(m_gradVel[dit()],
a_velFine[dit()],
m_gridsFine.get(dit()),
m_ebislFine[dit()],
a_lowOrderOneSidedGrad);
}
m_gradVel.exchange();
int ibox = 0;
for (DataIterator dit = m_gridsFine.dataIterator(); dit.ok(); ++dit)
{
EBFluxFAB& faceDiv = m_faceDivCell[dit()];
for (int idir = 0; idir < SpaceDim; idir++)
{
faceDivergence(faceDiv[idir],
m_gradVel[dit()],
a_velFine[dit()],
a_fluxVelFine[dit()],
m_gridsFine.get(dit()),
m_ebislFine[dit()],
idir);
}
ibox++;
}
m_faceDivCell.exchange();
//apply filter grid by grid
//the lambda gets multiplied in on the fly
//(the true tells grad div to do this multiplication)
for (DataIterator dit = m_gridsFine.dataIterator(); dit.ok(); ++dit)
{
EBCellFAB lambdaGradDivVel(m_ebislFine[dit()], m_gridsFine.get(dit()), SpaceDim);
gradDiv(lambdaGradDivVel,
m_gradVel[dit()],
a_velFine[dit()],
a_fluxVelFine[dit()],
m_faceDivCell[dit()],
m_gridsFine.get(dit()),
m_ebislFine[dit()],
dit(),
true,
a_noExtrapToCovered);
a_velFine[dit()] += lambdaGradDivVel;
}
}
//----------------------------------------------------------------------------
void
EBNormalizeByVolumeFraction::
operator()(LevelData<EBCellFAB>& a_Q,
const Interval& a_compInterval) const
{
CH_TIME("EBNormalizer::operator()");
// Endpoints of the given interval.
int begin = a_compInterval.begin(), end = a_compInterval.end(),
length = a_compInterval.size();
// Loop over the EBISBoxes within our grid. The EB data structures are
// indexed in the same manner as the non-EB data structures, so we piggy-
// back the former on the latter.
a_Q.exchange();
EBISLayout ebisLayout = m_levelGrid.getEBISL();
DisjointBoxLayout layout = m_levelGrid.getDBL();
for (DataIterator dit = layout.dataIterator(); dit.ok(); ++dit)
{
const EBISBox& box = ebisLayout[dit()];
EBCellFAB& QFAB = a_Q[dit()];
// Go over the irregular cells in this box.
const IntVectSet& irregCells = box.getIrregIVS(layout[dit()]);
// The average has to be computed from the uncorrected data from all
// the neighbors, so we can't apply the corrections in place. For now,
// we stash them in a map.
map<VolIndex, vector<Real> > correctedValues;
for (VoFIterator vit(irregCells, box.getEBGraph()); vit.ok(); ++vit)
{
Real kappajSum = 0.0;
vector<Real> kappajQjSum(length, 0.0);
// Get all of the indices of the VoFs within a monotone path
// radius of 1.
VolIndex vofi = vit();
Vector<VolIndex> vofjs;
EBArith::getAllVoFsInMonotonePath(vofjs, vofi, box, 1);
// Accumulate the contributions from the neighboring cells.
for (unsigned int j = 0; j < vofjs.size(); ++j)
{
VolIndex vofj = vofjs[j];
Real kappaj = box.volFrac(vofj);
for (int icomp = begin; icomp <= end; ++icomp)
{
kappajQjSum[icomp] += QFAB(vofj, icomp);
}
// Add this volume fraction to the sum.
kappajSum += kappaj;
}
if (kappajSum > 0.)
{
// Normalize the quantity and stow it.
vector<Real> correctedValue(length);
// Real kappai = box.volFrac(vofi); //unused dtg
for (int icomp = begin; icomp <= end; ++icomp)
{
// correctedValue[icomp - begin] =
// QFAB(vofi, icomp) + (1.0 - kappai) * kappajQjSum[icomp] / kappajSum;
correctedValue[icomp - begin] = kappajQjSum[icomp] / kappajSum;
}
correctedValues[vofi] = correctedValue;
}
}
// Apply the corrections.
for (map<VolIndex, vector<Real> >::const_iterator
cit = correctedValues.begin(); cit != correctedValues.end(); ++cit)
{
for (int icomp = begin; icomp <= end; ++icomp)
{
QFAB(cit->first, icomp) = cit->second[icomp-begin];
}
}
}
}
// ---------------------------------------------------------
void
NodeCoarseAverage::averageToCoarse(LevelData<NodeFArrayBox>& a_coarse,
LevelData<NodeFArrayBox>& a_fine)
{
// this is the method that is called externally.
// a_fine must have ghost nodes to depth m_refRatio/2.
// ghost data is required when averaging onto a fine-fine interface.
CH_assert(is_defined);
CH_assert(a_fine.ghostVect() >= (m_refRatio / 2) * IntVect::Unit);
//CH_assert(a_domainCoarse == m_domainCoarse);
a_fine.exchange(a_fine.interval());
DataIterator ditc = m_coarsenedGrids.dataIterator();
// initialize m_coarsenedFine with zero (on boxes in this proc)
// if the grids are different.
if (! m_sameGrids)
for (ditc.begin(); ditc.ok(); ++ditc)
m_coarsenedFine[ditc()].getFab().setVal(0.0);
// set m_coarsenedFine on interior nodes of m_coarsenedGrids to
// weighted average of fine nodes in neighborhood.
for (ditc.begin(); ditc.ok(); ++ditc)
{
// coarsened_fine, fine are cell-centered but represent data on nodes.
BaseFab<Real>& coarseFab = (m_sameGrids) ?
a_coarse[ditc()].getFab() : m_coarsenedFine[ditc()].getFab();
const BaseFab<Real>& fineFab = a_fine[ditc()].getFab();
// idea for setting m_coarsenedFine on interior nodes:
// (1) set m_coarsenedFine in interior of this box
// (interior of this box is another box).
// (2) then set m_coarsenedFine at other interior points.
// ---
// (1)
// set m_coarsenedFine at inner nodes of box
// (note coarsenedFineFab.box(), not m_coarsenedFine.box(),
// because we use nodes)
const Box thisBox = m_coarsenedGrids.get(ditc());
// inner: cell-centered box, indices correspond to inner nodes
Box inner(thisBox);
for (int idir = 0; idir < CH_SPACEDIM; idir++)
inner.growLo(idir, -1);
if (!inner.isEmpty())
FORT_NODEAVERAGE(CHF_FRA(coarseFab),
CHF_CONST_FRA(fineFab),
CHF_BOX(inner),
CHF_CONST_INT(m_refRatio),
CHF_CONST_FRA(m_weights));
// (2)
// remove inner nodes from interior, leaving only grid-boundary nodes.
// cell indices of innerCells are node indices of interior nodes.
// now do averaging for each interior boundary node.
Vector<IntVectSet>& IVSvec = m_IVSVsame[ditc()];
BitSet& fullvec = m_IVSVfull[ditc()];
for (int lcomp = 0; lcomp < IVSvec.size(); lcomp++)
{
IntVectSet& IVS = IVSvec[lcomp];
if (fullvec[lcomp])
{
Box container(IVS.minBox());
FORT_NODEAVERAGE(CHF_FRA(coarseFab),
CHF_CONST_FRA(fineFab),
CHF_BOX(container),
CHF_CONST_INT(m_refRatio),
CHF_CONST_FRA(m_weights));
}
else
for (IVSIterator it(IVS); it.ok(); ++it)
{
IntVect iv = it();
// Set coarseFab(iv, :) to average of fineFab(iv+m_refbox, :)
// Use either Fortran or C++.
// petermc, 23 Jul 2003: Fortran method is found to be faster,
// by a factor of 2.15 for total time on averageToCoarse
// function.
// begin Fortran method
FORT_NODEAVERAGEPOINT(CHF_FRA(coarseFab),
CHF_CONST_FRA(fineFab),
CHF_CONST_INTVECT(iv),
CHF_CONST_INT(m_refRatio),
CHF_CONST_FRA(m_weights));
// end Fortran method
// begin C++ method
// for (int ivar = 0; ivar < m_numcomps; ivar++)
// {
// Real csum = 0.;
// for (BoxIterator offb(m_refbox); offb.ok(); ++offb)
// {
// IntVect ivOff = offb();
// IntVect ivFine = m_refRatio*iv + ivOff;
// csum += m_weights(ivOff, 0) * fineFab(ivFine, ivar);
// }
// coarseFab(iv, ivar) = csum;
// end C++ method
}
}
}
// copy data at interior nodes of m_coarsenedFine to a_coarse data:
// see NodeFArrayBox for code.
// IF the grids are not the same, then
// copy data at interior nodes of m_coarsenedFine to a_coarse:
// see NodeFArrayBox for code.
if (! m_sameGrids)
{
copyInteriorNodes(a_coarse, m_coarsenedFine, m_IVSV);
}
// copyInteriorNodes(a_coarse, m_coarsenedFine, m_domainCoarse);
}
void
EBGradDivFilter::
gradDiv(LevelData<EBCellFAB>& a_gradDivVel,
LevelData<EBCellFAB>& a_velFine,
const LevelData<EBFluxFAB>& a_fluxVelFine,
const LevelData<EBCellFAB>* a_velCoar,
bool a_lowOrderOneSidedGrad,
bool a_noExtrapToCovered)
{
CH_TIME("EBGradDivFilter::gradDiv");
CH_assert(a_velFine.nComp() == SpaceDim);
Interval velInterv(0, SpaceDim-1);
//do coarse-fine interpolation if necessary
if (m_hasCoarser)
{
CH_assert(a_velCoar != NULL);
CH_assert(a_velCoar->nComp() == SpaceDim);
m_tensorCFI->coarseFineInterp(a_velFine, m_gradVel, *a_velCoar);
}
//fill in ghost cells
a_velFine.exchange(velInterv);
//compute gradient of velocity for parts NOT in ghost cells
for (DataIterator dit = m_gridsFine.dataIterator(); dit.ok(); ++dit)
{
m_gradVel[dit()].setVal(0.);
gradVel(m_gradVel[dit()],
a_velFine[dit()],
m_gridsFine.get(dit()),
m_ebislFine[dit()],
a_lowOrderOneSidedGrad);
}
m_gradVel.exchange();
int ibox = 0;
for (DataIterator dit = m_gridsFine.dataIterator(); dit.ok(); ++dit)
{
EBFluxFAB& fluxVel = m_faceDivCell[dit()];
for (int idir = 0; idir < SpaceDim; idir++)
{
faceDivergence(fluxVel[idir],
m_gradVel[dit()],
a_velFine[dit()],
a_fluxVelFine[dit()],
m_gridsFine.get(dit()),
m_ebislFine[dit()],
idir);
}
ibox++;
}
m_faceDivCell.exchange();
for (DataIterator dit = m_gridsFine.dataIterator(); dit.ok(); ++dit)
{
//the false tells gradDiv to not premultiply by lambda
gradDiv(a_gradDivVel[dit()],
m_gradVel[dit()],
a_velFine[dit()],
a_fluxVelFine[dit()],
m_faceDivCell[dit()],
m_gridsFine.get(dit()),
m_ebislFine[dit()],
dit(),
false,
a_noExtrapToCovered);
}
}
int
testIFFAB(const DisjointBoxLayout& a_dbl,
const EBISLayout & a_ebisl,
const Box & a_domain,
const Real & a_dx )
{
int faceDir = 0;
int nFlux = 1;
LayoutData<IntVectSet> irregSetsGrown;
LevelData< BaseIFFAB<Real> > fluxInterpolant;
EBArith::defineFluxInterpolant(fluxInterpolant,
irregSetsGrown,
a_dbl, a_ebisl, a_domain, nFlux, faceDir);
//set source fab to right ans over set only on grids interior cells
int ibox = 0;
for (DataIterator dit = a_dbl.dataIterator(); dit.ok(); ++dit)
{
BaseIFFAB<Real>& srcFab = fluxInterpolant[dit()];
srcFab.setVal(-1.0);
IntVectSet ivsSmall = irregSetsGrown[dit()];
const Box& grid = a_dbl.get(dit());
ivsSmall &= grid;
for (FaceIterator faceit(ivsSmall, a_ebisl[dit()].getEBGraph(), faceDir, FaceStop::SurroundingWithBoundary);
faceit.ok(); ++faceit)
{
srcFab(faceit(), 0) = rightAns(faceit());
}
ibox++;
}
//diagnostics
if (g_diagnosticMode)
{
pout() << " diagnostics for processor " << procID() << endl;
for (DataIterator dit = a_dbl.dataIterator(); dit.ok(); ++dit)
{
const IntVectSet& ivsGrown = irregSetsGrown[dit()];
const Box& grid = a_dbl.get(dit());
pout() << "============" << endl;
pout() << " box = " << grid;
pout() << ", full ivs = " ;
dumpIVS(&ivsGrown);
pout() << "============" << endl;
for (LayoutIterator lit = a_dbl.layoutIterator(); lit.ok(); ++lit)
{
const Box& grid2 = a_dbl.get(lit());
IntVectSet ivsIntersect = ivsGrown;
ivsIntersect &= grid2;
pout() << "intersection with box " << grid2 << " = ";
dumpIVS(&ivsIntersect);
pout() << "============" << endl;
}
}
}
BaseIFFAB<Real>::setVerbose(true);
//do the evil exchange
Interval interv(0, nFlux-1);
fluxInterpolant.exchange(interv);
ibox = 0;
//check the answer over grown set
Real tolerance = 0.001;
for (DataIterator dit = a_dbl.dataIterator(); dit.ok(); ++dit)
{
const BaseIFFAB<Real>& srcFab = fluxInterpolant[dit()];
const IntVectSet& ivsGrown = irregSetsGrown[dit()];
for (FaceIterator faceit(ivsGrown, a_ebisl[dit()].getEBGraph(), faceDir, FaceStop::SurroundingWithBoundary);
faceit.ok(); ++faceit)
{
Real correct = rightAns(faceit());
Real fabAns = srcFab(faceit(), 0);
if (Abs(correct - fabAns) > tolerance)
{
pout() << "iffab test failed at face "
<< faceit().gridIndex(Side::Lo)
<< faceit().gridIndex(Side::Hi) << endl;
pout() << " right ans = " << correct << endl;
pout() << " data holds= " << fabAns << endl;
int eekflag = -3;
return eekflag;
}
}
ibox++;
}
return 0;
}
void
EBLevelAdvect::
advectToFacesBCG(LevelData< EBFluxFAB >& a_extrapState,
const LevelData< EBCellFAB >& a_consState,
const LevelData< EBCellFAB >& a_normalVel,
const LevelData< EBFluxFAB >& a_advectionVel,
const LevelData< EBCellFAB >* a_consStateCoarseOld,
const LevelData< EBCellFAB >* a_consStateCoarseNew,
const LevelData< EBCellFAB >* a_normalVelCoarseOld,
const LevelData< EBCellFAB >* a_normalVelCoarseNew,
const Real& a_timeCoarseOld,
const Real& a_timeCoarseNew,
const Real& a_timeFine,
const Real& a_dt,
const LevelData<EBCellFAB>* const a_source,
const LevelData<EBCellFAB>* const a_sourceCoarOld,
const LevelData<EBCellFAB>* const a_sourceCoarNew)
{
CH_TIME("EBLevelAdvect::advectToFacesBCG (level)");
CH_assert(isDefined());
//create temp data with the correct number of ghost cells
IntVect ivGhost = m_nGhost*IntVect::Unit;
Interval consInterv(0, m_nVar-1);
Interval intervSD(0, SpaceDim-1);
// LevelData<EBCellFAB>& consTemp = (LevelData<EBCellFAB>&) a_consState;
// LevelData<EBCellFAB>& veloTemp = (LevelData<EBCellFAB>&) a_normalVel;
EBCellFactory factory(m_thisEBISL);
LevelData<EBCellFAB> consTemp(m_thisGrids, m_nVar, ivGhost, factory);
LevelData<EBCellFAB> veloTemp(m_thisGrids, SpaceDim, ivGhost, factory);
for (DataIterator dit = m_thisGrids.dataIterator(); dit.ok(); ++dit)
{
consTemp[dit()].setVal(0.);
}
a_consState.copyTo(consInterv, consTemp, consInterv);
a_normalVel.copyTo(intervSD, veloTemp, intervSD);
// Fill ghost cells using fillInterp, and copyTo.
if (m_hasCoarser)
{
CH_TIME("fillPatch");
m_fillPatch.interpolate(consTemp,
*a_consStateCoarseOld,
*a_consStateCoarseNew,
a_timeCoarseOld,
a_timeCoarseNew,
a_timeFine,
consInterv);
m_fillPatchVel.interpolate(veloTemp,
*a_normalVelCoarseOld,
*a_normalVelCoarseNew,
a_timeCoarseOld,
a_timeCoarseNew,
a_timeFine,
intervSD);
}
// Exchange all the data between grids
{
CH_TIME("initial_exchange");
consTemp.exchange(consInterv);
veloTemp.exchange(intervSD);
}
LevelData<EBCellFAB>* srcTmpPtr = NULL;
if (a_source != NULL)
{
// srcTmpPtr = (LevelData<EBCellFAB>*) a_source;
srcTmpPtr = new LevelData<EBCellFAB>(m_thisGrids, m_nVar, ivGhost, factory);
for (DataIterator dit = m_thisGrids.dataIterator(); dit.ok(); ++dit)
{
(*srcTmpPtr)[dit()].setVal(0.);
}
a_source->copyTo(consInterv, *srcTmpPtr, consInterv);
if ( (a_sourceCoarOld != NULL) &&
(a_sourceCoarNew != NULL) &&
(m_hasCoarser) )
{
CH_TIME("fillPatch");
m_fillPatch.interpolate(*srcTmpPtr,
*a_sourceCoarOld,
*a_sourceCoarNew,
a_timeCoarseOld,
a_timeCoarseNew,
a_timeFine,
consInterv);
{
CH_TIME("initial_exchange");
srcTmpPtr->exchange(consInterv);
}
}
}
{
CH_TIME("advectToFaces");
int ibox = 0;
for (DataIterator dit = m_thisGrids.dataIterator(); dit.ok(); ++dit)
{
const Box& cellBox = m_thisGrids.get(dit());
const EBISBox& ebisBox = m_thisEBISL[dit()];
if (!ebisBox.isAllCovered())
{
//the viscous term goes into here
EBCellFAB dummy;
EBCellFAB* srcPtr = &dummy;
if (srcTmpPtr != NULL)
{
srcPtr = (EBCellFAB*)(&((*srcTmpPtr)[dit()]));
}
const EBCellFAB& source = *srcPtr;
//unused in this case
BaseIVFAB<Real> boundaryPrim;
bool doBoundaryPrim = false;
EBFluxFAB& extrapFAB = a_extrapState[dit()];
advectToFacesBCG(extrapFAB,
boundaryPrim,
consTemp[dit()],
veloTemp[dit()],
a_advectionVel[dit()],
cellBox,
ebisBox,
a_dt,
a_timeFine,
source,
dit(),
doBoundaryPrim);
ibox++;
}
}
}
if (srcTmpPtr != NULL)
{
delete srcTmpPtr;
}
}
void getError(LevelData<double, 1> & a_error,
double & a_maxError,
const double & a_dx)
{
int rank, nprocs;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
MPI_Comm_size (MPI_COMM_WORLD, &nprocs);
std::cout << "I am rank " << rank << " of " << nprocs << " processes" << std::endl;
BoxLayout* layout = new BoxLayout();
int npatches;
int nPatchPerProc, iStartIdx, iEndIdx;
int patchID = -1;
double local_maxError = 0;
LevelData<double, 1> *phi;
LevelData<double, 1> *lphcalc;
LevelData<double, 1> *lphexac;
if(rank == 0)
{
*layout = a_error.getBoxLayout();
npatches = layout->size();
phi = new LevelData<double, 1>(*layout, s_nghost);
lphcalc = new LevelData<double, 1>(*layout, 0);
lphexac = new LevelData<double, 1>(*layout, 0);
// cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl;
initialize(*phi,*lphexac, a_dx);
//set ghost cells of phi
phi->exchange();
}
if(nprocs > 1)
MPI_Bcast( &npatches, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
if(npatches == 1)
nPatchPerProc = 1;
else
nPatchPerProc = npatches/nprocs;
iStartIdx = rank * nPatchPerProc;
iEndIdx = (npatches < (rank+1)*nPatchPerProc) ? npatches-1 : ((rank+1)*nPatchPerProc-1);
nPatchPerProc = iEndIdx - iStartIdx + 1;
std::cout << "I am rank " << rank << " working from " << iStartIdx << " to " << iEndIdx << std::endl;
// barrier to sync halo exchange
if(rank == 0)
{
Box bxdst, bxsrc;
MPI_Request reqs[5];
MPI_Status status[5];
int iwait = 0;
for(BLIterator blit(*layout); blit != blit.end(); ++blit)
{
//bxdst= (*layout)[*blit];
patchID++;
bool mypatch = patchID >= iStartIdx && patchID <=iEndIdx;
RectMDArray<double>& phiex = (*phi)[patchID];
RectMDArray<double>& lphca = (*lphcalc)[patchID];
RectMDArray<double>& lphex = (*lphexac)[patchID];
RectMDArray<double>& error = a_error[patchID];
bxsrc = phiex . getBox();
bxdst = lphca . getBox();
lphca.setVal(0.);
if(mypatch)
{
cout << "Rank 0 is working on patch " << patchID << endl;
double tmp;
tmp = doWork(bxdst, phiex, lphca, lphex, error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
else
{
int dest = patchID / nPatchPerProc;
// MPI_Isend(&phiex, sizeof(RectMDArray<double>),MPI_CHAR, dest, 0, MPI_COMM_WORLD,&reqs[0]);
//cout << "master send out patch " << patchID<< " sourceDataPointer with size " << phiex.getBox().sizeOf() << endl;
// MPI_Isend(&lphca, sizeof(RectMDArray<double>),MPI_CHAR, dest, 1, MPI_COMM_WORLD,&reqs[1]);
//cout << "master send out patch " << patchID<< " destinationDataPointer with size " << lphca.getBox().sizeOf() << endl;
MPI_Isend(&bxsrc, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[0]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
MPI_Isend(&bxdst, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[1]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
double *sourceDataPointer = phiex . getPointer();
double *destinationDataPointer = lphca . getPointer();
MPI_Isend(sourceDataPointer, phiex.getBox().sizeOf(),MPI_DOUBLE, dest, 2, MPI_COMM_WORLD,&reqs[2]);
MPI_Isend(destinationDataPointer, lphca.getBox().sizeOf(),MPI_DOUBLE, dest, 3, MPI_COMM_WORLD,&reqs[3]);
iwait++;
MPI_Waitall(4,reqs,status);
cout << "Rank 0 is sending patch " << patchID << " to rank " << dest << endl;
}
}
}
else
{
if(npatches == 1)
nPatchPerProc = 0;
else
nPatchPerProc = npatches/nprocs;
int src = 0;
if(nPatchPerProc > 0)
{
// MPI_Request reqs[5];
MPI_Status status[5];
Box bxdst, bxsrc;
RectMDArray<double>* phiex = new RectMDArray<double>();
RectMDArray<double>* lphca = new RectMDArray<double>();
RectMDArray<double>* lphex = new RectMDArray<double>();
RectMDArray<double>* error = new RectMDArray<double>();
int idx;
for(idx = 0; idx < nPatchPerProc; idx++)
{
// Box bxdst=((const BoxLayout&) layout)[*blit];
patchID++;
// MPI_Recv(phiex, sizeof(RectMDArray<double>), MPI_CHAR, src, 0, MPI_COMM_WORLD, &status[0]);
// MPI_Recv(lphca, sizeof(RectMDArray<double>), MPI_CHAR, src, 1, MPI_COMM_WORLD, &status[1]);
// double *sourceDataPointer = (double*) malloc(sizeof(double)*phiex->getBox().sizeOf());
MPI_Recv(&bxsrc, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[0]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
MPI_Recv(&bxdst, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[1]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
phiex->define(bxsrc);
lphca->define(bxdst);
lphex->define(bxdst);
error->define(bxdst);
MPI_Recv(phiex -> getPointer(), phiex->getBox().sizeOf(), MPI_DOUBLE, src, 2, MPI_COMM_WORLD, &status[2]);
cout << "Rank " << rank << " receive patch " << idx << " sourceDataPointer with size " << phiex->getBox().sizeOf() << endl;
// double *destinationDataPointer = (double*) malloc(sizeof(double)*lphca->getBox().sizeOf());
MPI_Recv(lphca -> getPointer(), lphca->getBox().sizeOf(), MPI_DOUBLE, src, 3, MPI_COMM_WORLD, &status[3]);
cout << "Rank " << rank << " receive patch " << idx << " destinationDataPointer with size " << lphca->getBox().sizeOf() << endl;
cout << "Rank " << rank << " is working on patch " << npatches << endl;
double tmp;
tmp = doWork(bxdst, *phiex, *lphca, *lphex, *error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
}
}
MPI_Allreduce(&local_maxError, &a_maxError, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if(rank == 0)
cout << "max Error is: " << a_maxError << endl;
}
