void refine(BoxLayout& a_output, const BoxLayout& a_input, int a_refinement)
{
if (!a_input.isClosed())
{
MayDay::Error("input to refine must be called with closed BoxLayout");
}
if (a_output.isClosed())
{
MayDay::Error("output of refine must be called on open BoxLayout");
}
a_output.deepCopy(a_input);
for (int ivec = 0; ivec < a_output.m_boxes->size(); ivec++)
{
(*a_output.m_boxes)[ivec].box.refine(a_refinement);
}
a_output.close();
}
// -------------------------------------------------------------
void
Mask::buildMask(BaseFab<int>& a_mask, const ProblemDomain& a_dProblem,
const BoxLayout& a_grids, const BoxLayout* a_fineGridsPtr,
int a_nRefFine)
{
// first set entire box to Physical BC
a_mask.setVal(maskPhysical);
// now set all of domain interior to coarse
Box domainInterior(a_mask.box());
domainInterior &= a_dProblem;
a_mask.setVal(maskCoarse,domainInterior,0);
// now loop over this level's boxes and set them to "copy"
LayoutIterator lit = a_grids.layoutIterator();
for (lit.reset(); lit.ok(); ++lit)
{
Box intersectBox = a_grids.get(lit());
intersectBox &= a_mask.box();
if (!intersectBox.isEmpty())
{
a_mask.setVal(maskCopy,intersectBox,0);
}
}
// if finer grids exist, set them to "covered"
if (a_fineGridsPtr != NULL)
{
CH_assert (a_nRefFine > 1);
LayoutIterator litFine = a_fineGridsPtr->layoutIterator();
for (litFine.reset(); litFine.ok(); ++litFine)
{
Box coarsenedBox(a_fineGridsPtr->get(litFine()));
coarsenedBox.coarsen(a_nRefFine);
coarsenedBox &= a_mask.box();
if (!coarsenedBox.isEmpty())
{
a_mask.setVal(maskCovered,coarsenedBox,0);
}
}
}
}
/* ECMAScript constructor */
Handle<Value> BoxLayout::New(const Arguments& args)
{
HandleScope scope;
if (!args.IsConstructCall()) {
return ThrowException(Exception::TypeError(
String::New("Use the new operator to create instances of this object."))
);
}
// Creates a new instance object of this type and wraps it.
BoxLayout* obj = new BoxLayout();
obj->Wrap(args.This());
/* Initializing sub-objects */
Local<Object> ScrollableObject = Scrollable::New(MX_SCROLLABLE(obj->_actor));
args.Holder()->Set(String::NewSymbol("scroll"), ScrollableObject);
return scope.Close(args.This());
}
void testlap(LevelData<double >& a_phi, double a_dx,char* a_str)
{
double coef = 1./(a_dx*a_dx);
Stencil<double> Laplacian(make_pair(getZeros(),-DIM*2*coef));
BoxLayout bl = a_phi.getBoxLayout();
for (int dir = 0; dir < DIM ; dir++)
{
Point edir = getUnitv(dir);
Stencil<double> plus(make_pair(Shift(edir),coef));
Stencil<double> minus(make_pair(Shift(edir*(-1)),coef));
Laplacian = Laplacian + minus + plus;
}
a_phi.exchange();
RectMDArray<double> LPhi00(bl.getDomain());
for (BLIterator blit(bl); blit != blit.end(); ++blit)
{
LPhi00 |= Laplacian(a_phi[*blit],bl[*blit]);
}
MDWrite(a_str,LPhi00);
};
void
coarsen(BoxLayout& a_output, const BoxLayout& a_input, int a_refinement)
{
if (!a_input.isClosed())
{
MayDay::Error("input to coarsen must be called with closed BoxLayout");
}
if (a_output.isClosed())
{
MayDay::Error("output of coarsen must be called on open BoxLayout");
}
//a_output.deepCopy(a_input);
a_output.m_boxes = RefCountedPtr<Vector<Entry> >(new Vector<Entry>(*(a_input.m_boxes)));
a_output.m_layout = a_input.m_layout;
#ifdef CH_MPI
a_output.m_dataIndex = a_input.m_dataIndex;
#endif
for (int ivec = 0; ivec < a_output.m_boxes->size(); ivec++)
{
(*a_output.m_boxes)[ivec].box.coarsen(a_refinement);
}
a_output.close();
}
//checks equality of the vector of boxes inside m_boxes
bool BoxLayout::sameBoxes(const BoxLayout& a_layout) const
{
bool retval;
if (size() == a_layout.size())
{
retval = true;
for (int iBox = 0; iBox < size(); ++iBox)
{
//RefCountedPtr<Vector<Entry> > m_boxes;
if ((*m_boxes)[iBox].box != (*a_layout.m_boxes)[iBox].box)
{
retval = false;
}
}
}
else
{
retval = false;
}
return retval;
}
void ReductionCopier::define(const BoxLayout& a_level,
const BoxLayout& a_dest,
const ProblemDomain& a_domain,
const IntVect& a_ghost,
const Vector<int>& a_transverseDir,
bool a_exchange)
{
#ifdef MULTIDIM_TIMER
CH_TIME("ReductionCopier::define")
#endif
m_isDefined = true;
m_transverseDir = a_transverseDir;
CH_assert(a_level.isClosed());
CH_assert(a_dest.isClosed());
// CH_assert(a_level.checkPeriodic(a_domain));
clear();
buffersAllocated = false;
//bool self = a_dest == a_level;
const BoxLayout& level= a_level;
const BoxLayout& dest = a_dest;
// note that while much of the implementation for the
// periodic case remains in this function (as copied, more
// or less, from the original Copier), at least at the moment
// periodic support is "turned off" for the ReductionCopier
// because it's not entirely clear it will do the right thing
// for this case. I've left the vast majority of the
// periodic implementation in place, however, in case we
// change our minds... (DFM 3/10/09)
// set up vector of dataIndexes to keep track of which
// "to" boxes are not completely contained within the primary
// domain. these boxes are then candidates for filling by
// periodic images of the "from" data.
Vector<DataIndex> periodicallyFilledToVect;
// in order to cull which "from" data may be needed to
// fill the "to" data, keep track of the radius around the
// primary domain in which all these cells lie.
// do this by incrementally growing the domain box and
// keeping track of what this radius is.
// just to make things simpler, start off with a radius of one
Box grownDomainCheckBox = a_domain.domainBox();
grownDomainCheckBox.grow(1);
int periodicCheckRadius = 1;
// since valid regions of the "from" DBL may also be outside
// the primary domain, need to keep track of whether any of these
// need to be checked separately.
Vector<DataIndex> periodicFromVect;
// use same domain trick here as well
Box grownFromDomainCheckBox = a_domain.domainBox();
int periodicFromCheckRadius = 1;
Box domainBox(a_domain.domainBox());
// grab index extents of domain in transverse direction
Vector<int> transverseLo(m_transverseDir.size());
Vector<int> transverseHi(m_transverseDir.size());
for (int n=0; n<m_transverseDir.size(); n++)
{
// grab index extents of domain in transverse direction
transverseLo[n] = domainBox.smallEnd(m_transverseDir[n]);
transverseHi[n] = domainBox.bigEnd(m_transverseDir[n]);
}
bool isPeriodic = false;
// I think the right thing here is that the ReductionCopier
// completely ignores periodicity
//if (!domainBox.isEmpty())
//isPeriodic = a_domain.isPeriodic();
// (dfm -- 9/13/05) as currently written, the Copier won't correctly
// handle periodic cases where the number of ghost cells is greater
// than the width of the domain. We _should_ do multiple wraparounds,
// but we don't. So, put in this assertion. We can revisit this if it
// becomes an issue
if (isPeriodic)
{
for (int dir=0; dir<SpaceDim; dir++)
{
if (a_domain.isPeriodic(dir))
{
CH_assert (a_ghost[dir] <= domainBox.size(dir));
}
}
}
unsigned int myprocID = procID();
// The following 4 for loops are the result of a performance optimization.
// When increasing the size of the problem, we found that the code was
// looping over every destination box for every source box which was N1*N2
// loop iterations (essentially an N-squared approach).
// The following code attempts to simply reduce N1 and N2 by first separating
// the boxes (or LayoutIndexes to boxes) that reside on the current processor.
// Then the loop to determine which boxes of the first list intersect with
// which boxes of the second list can be done in N1' * N2' iterations,
// where N1' is the reduced N1 and N2' is the reduced N2.
// We have to break up the assigning of MotionItems into two separate
// loops and be careful about the local copies. These 4 loops are
// significantly faster than the original for loop -- _especially_
// for large problems. (ndk)
#ifdef CH_MPI // don't need to do this in serial
// make a vector of boxes (or LayoutIndexes to boxes) from destination layout
// that are known to reside on this processor.
vector<DataIndex> vectorDestDI;
vector<DataIndex> vectorDestOnProcDI;
for (LayoutIterator to(a_dest.layoutIterator()); to.ok(); ++to)
{
vectorDestDI.push_back(DataIndex(to()));
if (myprocID == dest.procID(to()))
{
vectorDestOnProcDI.push_back(DataIndex(to()));
}
}
// make a vector of boxes (or LayoutIndexes to boxes) from "level"/src layout
// that are known to reside on this processor.
vector<DataIndex> vectorLevelDI;
vector<DataIndex> vectorLevelOnProcDI;
for (LayoutIterator from(a_level.layoutIterator()); from.ok(); ++from)
{
vectorLevelDI.push_back(DataIndex(from()));
if (myprocID == level.procID(from()))
{
vectorLevelOnProcDI.push_back(DataIndex(from()));
}
}
#else
// in serial, it's not very interesting as it's all of them.
vector<DataIndex> vectorDestOnProcDI;
vector<DataIndex> vectorLevelDI;
for (LayoutIterator to(a_dest.layoutIterator()); to.ok(); ++to)
{
vectorDestOnProcDI.push_back(DataIndex(to()));
}
for (LayoutIterator from(a_level.layoutIterator()); from.ok(); ++from)
{
vectorLevelDI.push_back(DataIndex(from()));
}
#endif
// loop over all dest/to DI's on my processor
for (vector<DataIndex>::iterator vdi=vectorDestOnProcDI.begin();
vdi != vectorDestOnProcDI.end(); ++vdi)
{
// at this point, i know myprocID == toProcID
const DataIndex todi(*vdi);
Box ghost(dest[todi]);
// don't do anything if ghost is entirely outside domain
// may need to revisit this in the periodic case
if (a_domain.intersects(ghost))
{
// this is somewhat different from the "normal" Copier. We only want
// to look at ghost cells if they're outside the domain.
// note that we don't want to do this for domain ghost cells in the
// transverse (spreading) directions.
for (int dir=0; dir<SpaceDim; dir++)
{
bool isTransverseDir = false;
for (int n=0; n<m_transverseDir.size(); n++)
{
if (dir == m_transverseDir[n]) isTransverseDir = true;
}
// only need to do this if we specify a ghost radius
if (!isTransverseDir && a_ghost[dir] != 0)
{
if (ghost.bigEnd(dir) == domainBox.bigEnd(dir))
{
ghost.growHi(dir, a_ghost[dir]);
}
if (ghost.smallEnd(dir) == domainBox.smallEnd(dir))
{
ghost.growLo(dir, a_ghost[dir]);
}
}
}
Box destBox(ghost);
// for this Copier, we want to grow the "dest" region to cover the
// entire domain in the transverse direction. In that way, we
// will be able to catch all of the source data regions which
// project onto the dest boxes
// do this for all transverse directions
for (int n=0; n<m_transverseDir.size(); n++)
{
ghost.setSmall(m_transverseDir[n], transverseLo[n]);
ghost.setBig(m_transverseDir[n], transverseHi[n]);
}
// then for each level/from DI, see if they intersect
for (vector<DataIndex>::iterator vli = vectorLevelDI.begin();
vli != vectorLevelDI.end(); ++vli)
{
const DataIndex fromdi(*vli);
const unsigned int fromProcID = level.procID(fromdi);
Box fromBox(level[fromdi]);
// also grow the fromBox if it's near the domain boundary and
// if there are ghost cells specified.
for (int dir=0; dir<SpaceDim; dir++)
{
bool isTransverseDir = false;
for (int n=0; n<m_transverseDir.size(); n++)
{
if (dir == m_transverseDir[n]) isTransverseDir = true;
}
// only need to do this if we specify a ghost radius
// and if it's not a transverseDir
if (!isTransverseDir && a_ghost[dir] != 0)
{
if (fromBox.bigEnd(dir) == domainBox.bigEnd(dir))
{
fromBox.growHi(dir, a_ghost[dir]);
}
if (fromBox.smallEnd(dir) == domainBox.smallEnd(dir))
{
fromBox.growLo(dir, a_ghost[dir]);
}
}
}
if (fromBox.bigEnd(0) < ghost.smallEnd(0))
{
//can skip rest cuz we haven't gotten to something interesting
continue;
}
if (ghost.intersectsNotEmpty(fromBox))
{
Box box(ghost); // ??
box&=fromBox; // ??
// to get correct toBox, stretch box in the transverse direction
// then intersect with the destBox to get the appropriate part
// of the destbox
Box toBox(box);
// do this for all transverse directions
for (int n=0; n<m_transverseDir.size(); n++)
{
toBox.setSmall(m_transverseDir[n], transverseLo[n]);
toBox.setBig(m_transverseDir[n], transverseHi[n]);
}
toBox &= destBox;
MotionItem* item = new (s_motionItemPool.getPtr())
MotionItem(fromdi, todi, box, toBox);
if (item == NULL)
{
MayDay::Error("Out of Memory in copier::define");
}
if (fromProcID == myprocID)
{
// local move
if (a_exchange && fromdi == todi)
s_motionItemPool.returnPtr(item);
else
m_localMotionPlan.push_back(item);
}
else
{
item->procID = fromProcID;
m_toMotionPlan.push_back(item);
}
}
if (fromBox.smallEnd(0) > ghost.bigEnd(0))
{
//can break out of loop, since we know that the smallEnd
// of all the remaining boxes are lexigraphically beyond this ghosted box.
break;
}
}
} // end if ghost intersects domain
}
// Don't need to worry about this in serial as we already
// took care of the local copy motion items just above. skip this.
#ifdef CH_MPI
// loop over all dest/to DI's
for (vector<DataIndex>::iterator vdi=vectorDestDI.begin();
vdi != vectorDestDI.end(); ++vdi)
{
const DataIndex todi(*vdi);
Box ghost(dest[todi]);
// don't do anything if ghost is outside domain
if (a_domain.intersects(ghost) )
{
// this is somewhat different from the "normal" Copier. We only want
// to look at ghost cells if they're outside the domain.
// note that we don't want to do this for domain ghost cells in the
// transverse (spreading) directions.
for (int dir=0; dir<SpaceDim; dir++)
{
bool isTransverseDir = false;
for (int n=0; n<m_transverseDir.size(); n++)
{
if (dir == m_transverseDir[n]) isTransverseDir = true;
}
// only need to do this if we specify a ghost radius
if (!isTransverseDir && a_ghost[dir] != 0)
{
if (ghost.bigEnd(dir) == domainBox.bigEnd(dir))
{
ghost.growHi(dir, a_ghost[dir]);
}
if (ghost.smallEnd(dir) == domainBox.smallEnd(dir))
{
ghost.growLo(dir, a_ghost[dir]);
}
}
}
Box destBox(ghost);
// for this Copier, we want to grow the "dest" region to cover the
// entire domain in the transverse direction. In that way, we
// will be able to catch all of the source data regions which
// project onto the dest boxes
// do this for all transverse directions
for (int n=0; n<m_transverseDir.size(); n++)
{
ghost.setSmall(m_transverseDir[n], transverseLo[n]);
ghost.setBig(m_transverseDir[n], transverseHi[n]);
}
const unsigned int toProcID = dest.procID(todi);
// then for each level/from DI on this processor, see if they intersect
for (vector<DataIndex>::iterator vli = vectorLevelOnProcDI.begin();
vli != vectorLevelOnProcDI.end(); ++vli)
{
// at this point, i know myprocID == fromProcID
const DataIndex fromdi(*vli);
Box fromBox(level[fromdi]);
// also grow the fromBox if it's near the domain boundary and
// if there are ghost cells specified.
for (int dir=0; dir<SpaceDim; dir++)
{
bool isTransverseDir = false;
for (int n=0; n<m_transverseDir.size(); n++)
{
if (dir == m_transverseDir[n]) isTransverseDir = true;
}
// only need to do this if we specify a ghost radius
// and if it's not a transverseDir
if (!isTransverseDir && a_ghost[dir] != 0)
{
if (fromBox.bigEnd(dir) == domainBox.bigEnd(dir))
{
fromBox.growHi(dir, a_ghost[dir]);
}
if (fromBox.smallEnd(dir) == domainBox.smallEnd(dir))
{
fromBox.growLo(dir, a_ghost[dir]);
}
}
}
if (fromBox.bigEnd(0) < ghost.smallEnd(0))
{
//can skip rest cuz we haven't gotten to something interesting
continue;
}
if (ghost.intersectsNotEmpty(fromBox))
{
Box box(ghost); // ??
box&=fromBox; // ??
// to get correct toBox, stretch box in the transverse direction
// then intersect with the destBox to get the appropriate part
// of the destbox
// do this for each transverse direction
Box toBox(box);
for (int n=0; n<m_transverseDir.size(); n++)
{
toBox.setSmall(m_transverseDir[n], transverseLo[n]);
toBox.setBig(m_transverseDir[n], transverseHi[n]);
}
toBox &= destBox;
if (toProcID == myprocID)
{
// local move
// don't push back here! or you will get two.
// we already did it above...
//m_localMotionPlan.push_back(item);
}
else
{
MotionItem* item = new (s_motionItemPool.getPtr())
MotionItem(fromdi, todi, box, toBox);
if (item == NULL)
{
MayDay::Error("Out of Memory in copier::define");
}
item->procID = toProcID;
m_fromMotionPlan.push_back(item);
}
}
if (fromBox.smallEnd(0) > ghost.bigEnd(0))
{
//can break out of loop, since we know that the smallEnd
// of all the remaining boxes are lexigraphically beyond this ghosted box.
break;
}
}
} // end if ghost intersects domain
}
#endif
// put periodic intersection checking in here for "to" boxes
if (isPeriodic)
{
for (LayoutIterator to(a_dest.layoutIterator()); to.ok(); ++to)
{
Box ghost(dest[to()]);
// don't do anything if ghost doesn't intersect domain
if (a_domain.intersects(ghost) )
{
// this is somewhat different from the "normal" Copier. We only want
// to look at ghost cells if they're outside the domain.
// note that we don't want to do this for domain ghost cells in the
// transverse (spreading) directions.
for (int dir=0; dir<SpaceDim; dir++)
{
bool isTransverseDir = false;
for (int n=0; n<m_transverseDir.size(); n++)
{
if (dir == m_transverseDir[n]) isTransverseDir = true;
}
// only need to do this if we specify a ghost radius
if (!isTransverseDir && a_ghost[dir] != 0)
{
if (ghost.bigEnd(dir) == domainBox.bigEnd(dir))
{
ghost.growHi(dir, a_ghost[dir]);
}
if (ghost.smallEnd(dir) == domainBox.smallEnd(dir))
{
ghost.growLo(dir, a_ghost[dir]);
}
}
}
Box destBox(ghost);
// for this Copier, we want to grow the "dest" region to cover the
// entire domain in the transverse direction. In that way, we
// will be able to catch all of the source data regions which
// project onto the dest boxes
// do this for each transverse directions
for (int n=0; n<m_transverseDir.size(); n++)
{
ghost.setSmall(m_transverseDir[n], transverseLo[n]);
ghost.setBig(m_transverseDir[n], transverseHi[n]);
}
//unsigned int toProcID = dest.procID(to()); // unused variable
// only do this if ghost box hangs over domain edge
if (!domainBox.contains(ghost))
{
// add the dataIndex for this box to the list
// of boxes which we need to come back to
periodicallyFilledToVect.push_back(DataIndex(to()));
// now check to see if we need to grow the
// periodic check radius
if (!grownDomainCheckBox.contains(ghost))
{
// grow the domainCheckBox until it contains ghost
while (!grownDomainCheckBox.contains(ghost))
{
grownDomainCheckBox.grow(1);
periodicCheckRadius++;
}
} // end if we need to grow radius around domain
} //end if ghost box is not contained in domain
} // end if original ghost box didn't intersect domain
}// end if periodic
}
// Here ends the so-called N-squared optimizations. the rest is unchanged. (ndk)
// now do periodic checking, if necessary
if (isPeriodic)
{
// the only "from" boxes we will need to check
// will be those within periodicCheckRadius of the
// domain boundary. so, create a box to screen out
// those which we will need to check.
Box shrunkDomainBox = a_domain.domainBox();
shrunkDomainBox.grow(-periodicCheckRadius);
ShiftIterator shiftIt = a_domain.shiftIterator();
IntVect shiftMult(domainBox.size());
// now loop over "from" boxes
for (LayoutIterator from(a_level.layoutIterator()); from.ok(); ++from)
{
// first check to see whether we need to look at this box
const Box& fromBox = level[from()];
if (!shrunkDomainBox.contains(fromBox))
{
unsigned int fromProcID = level.procID(from());
// check to see if fromBox is contained in domain,
// if not, add it to the list of fromBoxes we need to
// go back and check separately to see if it will
// fill one of the "to" boxes
if (!domainBox.contains(fromBox))
{
periodicFromVect.push_back(DataIndex(from()));
if (!grownFromDomainCheckBox.contains(fromBox))
{
while (!grownFromDomainCheckBox.contains(fromBox))
{
grownFromDomainCheckBox.grow(1);
periodicFromCheckRadius++;
}
} // end if we need to grow domain check box
} // end if fromBox is outside domain
// now loop over those "to" boxes which were not contained
// in the domain
for (int toRef=0; toRef<periodicallyFilledToVect.size(); toRef++)
{
DataIndex toIndex = periodicallyFilledToVect[toRef];
unsigned int toProcID = dest.procID(toIndex);
// don't worry about anything that doesn't involve this proc
if (toProcID != myprocID && fromProcID != myprocID)
{
// do nothing
}
else
{
Box ghost(dest[toIndex]);
// don't do anything if ghost doesn't intersect domain
if (a_domain.intersects(ghost))
{
// this is somewhat different from the "normal" Copier.
// We only want
// to look at ghost cells if they're outside the domain.
// note that we don't want to do this for domain ghost
// cells in the transverse (spreading) directions.
for (int dir=0; dir<SpaceDim; dir++)
{
bool isTransverseDir = false;
for (int n=0; n<m_transverseDir.size(); n++)
{
if (dir == m_transverseDir[n]) isTransverseDir = true;
}
// only need to do this if we specify a ghost radius
if (!isTransverseDir && a_ghost[dir] != 0)
{
if (ghost.bigEnd(dir) == domainBox.bigEnd(dir))
{
ghost.growHi(dir, a_ghost[dir]);
}
if (ghost.smallEnd(dir) == domainBox.smallEnd(dir))
{
ghost.growLo(dir, a_ghost[dir]);
}
}
}
Box destBox(ghost);
// for this Copier, we want to grow the "dest"
// region to cover the entire domain in the transverse
// direction. In that way, we will be able to catch all
// of the source data regions which project onto the
// dest boxes
// do this for each transverse direction
for (int n=0; n<m_transverseDir.size(); n++)
{
ghost.setSmall(m_transverseDir[n], transverseLo[n]);
ghost.setBig(m_transverseDir[n], transverseHi[n]);
}
// now need to loop over shift vectors and look at images
for (shiftIt.begin(); shiftIt.ok(); ++shiftIt)
{
IntVect shiftVect(shiftIt()*shiftMult);
ghost.shift(shiftVect);
if (ghost.intersectsNotEmpty(fromBox)) // rarely happens
{
Box intersectBox(ghost);
intersectBox &= fromBox;
Box toBox(intersectBox);
toBox.shift(-shiftVect);
toBox &= destBox;
MotionItem* item = new (s_motionItemPool.getPtr())
MotionItem(DataIndex(from()), DataIndex(toIndex),
intersectBox, toBox);
if (item == NULL)
{
MayDay::Error("Out of Memory in copier::define");
}
if (toProcID == fromProcID) // local move
m_localMotionPlan.push_back(item);
else if (fromProcID == myprocID)
{
item->procID = toProcID;
m_fromMotionPlan.push_back(item);
}
else
{
item->procID = fromProcID;
m_toMotionPlan.push_back(item);
}
} // end if shifted box intersects
ghost.shift(-shiftVect);
} // end loop over shift vectors
} // end if original "ghost" intersected domain
} // end if either from box or to box are on this proc
} // end loop over destination boxes
} // end if source box is close to domain boundary
} // end loop over destination boxes
// now go back through the "from" boxes which were outside
// the domain and see if they intersect any toBoxes
if (periodicFromVect.size() != 0)
{
// the only "to" boxes we will need to check
// will be those within periodicCheckRadius of the
// domain boundary. so, create a box to screen out
// those which we will need to check.
shrunkDomainBox = a_domain.domainBox();
shrunkDomainBox.grow(-periodicFromCheckRadius);
// now loop over the "to" boxes
for (LayoutIterator to(a_dest.layoutIterator()); to.ok(); ++to)
{
// first check to see whether we need to look at this box
Box ghost(dest[to()]);
// don't do anything if "ghost" doesn't interesect domain
if (a_domain.intersects(ghost))
{
// this is somewhat different from the "normal" Copier.
// We only want
// to look at ghost cells if they're outside the domain.
// note that we don't want to do this for domain ghost cells
// in the transverse (spreading) directions.
for (int dir=0; dir<SpaceDim; dir++)
{
bool isTransverseDir = false;
for (int n=0; n<m_transverseDir.size(); n++)
{
if (dir == m_transverseDir[n]) isTransverseDir = true;
}
// only need to do this if we specify a ghost radius
if (!isTransverseDir && a_ghost[dir] != 0)
{
if (ghost.bigEnd(dir) == domainBox.bigEnd(dir))
{
ghost.growHi(dir, a_ghost[dir]);
}
if (ghost.smallEnd(dir) == domainBox.smallEnd(dir))
{
ghost.growLo(dir, a_ghost[dir]);
}
}
}
Box destBox(ghost);
// for this Copier, we want to grow the "dest" region to
// cover the entire domain in the transverse direction.
// In that way, we will be able to catch all of the source
// data regions which project onto the dest boxes
// do this for each transverse direction
for (int n=0; n<m_transverseDir.size(); n++)
{
ghost.setSmall(m_transverseDir[n], transverseLo[n]);
ghost.setBig(m_transverseDir[n], transverseHi[n]);
}
if (!shrunkDomainBox.contains(ghost))
{
unsigned int toProcID = a_dest.procID(to());
// now loop over those "from" boxes which are not
// contained by the domain
for (int fromRef = 0; fromRef<periodicFromVect.size(); fromRef++)
{
DataIndex fromIndex = periodicFromVect[fromRef];
const Box& fromBox = level[fromIndex];
unsigned int fromProcID = level.procID(fromIndex);
// don't worry about anything which doesn't involve
// this proc
if (toProcID != myprocID && fromProcID != myprocID)
{
// do nothing
}
else
{
// now need to loop over shift vectors and look at images
for (shiftIt.begin(); shiftIt.ok(); ++shiftIt)
{
IntVect shiftVect(shiftIt()*shiftMult);
ghost.shift(shiftVect);
if (ghost.intersectsNotEmpty(fromBox))
{
Box intersectBox(ghost);
intersectBox &= fromBox;
Box toBox(intersectBox);
toBox.shift(-shiftVect);
toBox &= destBox;
MotionItem* item = new (s_motionItemPool.getPtr())
MotionItem(DataIndex(fromIndex), DataIndex(to()),
intersectBox, toBox);
if (item == NULL)
{
MayDay::Error("Out of Memory in copier::define");
}
if (toProcID == fromProcID) // local move
m_localMotionPlan.push_back(item);
else if (fromProcID == myprocID)
{
item->procID = toProcID;
m_fromMotionPlan.push_back(item);
}
else
{
item->procID = fromProcID;
m_toMotionPlan.push_back(item);
}
} // end if shifted box intersects
ghost.shift(-shiftVect);
} // end loop over shift vectors
} // end if either from box or to box are on this proc
} // end loop over "from" boxes
} // end if destination box is close to domain boundary
} // end if original "ghost" intersects domain
} // end loop over destination boxes
} // end if any of the "From" boxes were outside the domain
} // end if we need to do anything for periodicity
}
Real DotProduct(const BoxLayoutData<FArrayBox>& a_dataOne,
const BoxLayoutData<FArrayBox>& a_dataTwo,
const BoxLayout& a_dblIn,
const Interval& a_comps)
{
Vector<Real> rhodot;
rhodot.reserve(10);
const int startcomp = a_comps.begin();
const int endcomp = a_comps.end();
//calculate the single-processor dot product
DataIterator dit = a_dataOne.dataIterator();
for (dit.reset(); dit.ok(); ++dit)
{
Box fabbox = a_dblIn.get(dit());
const FArrayBox& onefab = a_dataOne[dit()];
const FArrayBox& twofab = a_dataTwo[dit()];
CH_assert(onefab.box().contains(fabbox));
CH_assert(twofab.box().contains(fabbox));
Real dotgrid = 0;
FORT_DOTPRODUCT(CHF_REAL(dotgrid),
CHF_CONST_FRA(onefab),
CHF_CONST_FRA(twofab),
CHF_BOX(fabbox),
CHF_CONST_INT(startcomp),
CHF_CONST_INT(endcomp));
rhodot.push_back(dotgrid);
}
// now for the multi-processor fandango
//gather all the rhodots onto a vector and add them up
int baseProc = 0;
Vector<Vector<Real> >dotVec;
gather(dotVec, rhodot, baseProc);
Real rhodotTot = 0.0;
if (procID() == baseProc)
{
CH_assert(dotVec.size() == numProc());
rhodot.resize(a_dblIn.size());
int index = 0;
for (int p = 0; p < dotVec.size(); p++)
{
Vector<Real>& v = dotVec[p];
for (int i = 0; i < v.size(); i++, index++)
{
rhodot[index] = v[i];
}
}
rhodot.sort();
for (int ivec = 0; ivec < rhodot.size(); ivec++)
{
rhodotTot += rhodot[ivec];
}
}
//broadcast the sum to all processors.
broadcast(rhodotTot, baseProc);
//return the total
return rhodotTot;
}
void getError(LevelData<double, 1> & a_error,
double & a_maxError,
const double & a_dx)
{
int rank, nprocs;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
MPI_Comm_size (MPI_COMM_WORLD, &nprocs);
std::cout << "I am rank " << rank << " of " << nprocs << " processes" << std::endl;
BoxLayout* layout = new BoxLayout();
int npatches;
int nPatchPerProc, iStartIdx, iEndIdx;
int patchID = -1;
double local_maxError = 0;
LevelData<double, 1> *phi;
LevelData<double, 1> *lphcalc;
LevelData<double, 1> *lphexac;
if(rank == 0)
{
*layout = a_error.getBoxLayout();
npatches = layout->size();
phi = new LevelData<double, 1>(*layout, s_nghost);
lphcalc = new LevelData<double, 1>(*layout, 0);
lphexac = new LevelData<double, 1>(*layout, 0);
// cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl;
initialize(*phi,*lphexac, a_dx);
//set ghost cells of phi
phi->exchange();
}
if(nprocs > 1)
MPI_Bcast( &npatches, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
if(npatches == 1)
nPatchPerProc = 1;
else
nPatchPerProc = npatches/nprocs;
iStartIdx = rank * nPatchPerProc;
iEndIdx = (npatches < (rank+1)*nPatchPerProc) ? npatches-1 : ((rank+1)*nPatchPerProc-1);
nPatchPerProc = iEndIdx - iStartIdx + 1;
std::cout << "I am rank " << rank << " working from " << iStartIdx << " to " << iEndIdx << std::endl;
// barrier to sync halo exchange
if(rank == 0)
{
Box bxdst, bxsrc;
MPI_Request reqs[5];
MPI_Status status[5];
int iwait = 0;
for(BLIterator blit(*layout); blit != blit.end(); ++blit)
{
//bxdst= (*layout)[*blit];
patchID++;
bool mypatch = patchID >= iStartIdx && patchID <=iEndIdx;
RectMDArray<double>& phiex = (*phi)[patchID];
RectMDArray<double>& lphca = (*lphcalc)[patchID];
RectMDArray<double>& lphex = (*lphexac)[patchID];
RectMDArray<double>& error = a_error[patchID];
bxsrc = phiex . getBox();
bxdst = lphca . getBox();
lphca.setVal(0.);
if(mypatch)
{
cout << "Rank 0 is working on patch " << patchID << endl;
double tmp;
tmp = doWork(bxdst, phiex, lphca, lphex, error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
else
{
int dest = patchID / nPatchPerProc;
// MPI_Isend(&phiex, sizeof(RectMDArray<double>),MPI_CHAR, dest, 0, MPI_COMM_WORLD,&reqs[0]);
//cout << "master send out patch " << patchID<< " sourceDataPointer with size " << phiex.getBox().sizeOf() << endl;
// MPI_Isend(&lphca, sizeof(RectMDArray<double>),MPI_CHAR, dest, 1, MPI_COMM_WORLD,&reqs[1]);
//cout << "master send out patch " << patchID<< " destinationDataPointer with size " << lphca.getBox().sizeOf() << endl;
MPI_Isend(&bxsrc, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[0]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
MPI_Isend(&bxdst, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[1]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
double *sourceDataPointer = phiex . getPointer();
double *destinationDataPointer = lphca . getPointer();
MPI_Isend(sourceDataPointer, phiex.getBox().sizeOf(),MPI_DOUBLE, dest, 2, MPI_COMM_WORLD,&reqs[2]);
MPI_Isend(destinationDataPointer, lphca.getBox().sizeOf(),MPI_DOUBLE, dest, 3, MPI_COMM_WORLD,&reqs[3]);
iwait++;
MPI_Waitall(4,reqs,status);
cout << "Rank 0 is sending patch " << patchID << " to rank " << dest << endl;
}
}
}
else
{
if(npatches == 1)
nPatchPerProc = 0;
else
nPatchPerProc = npatches/nprocs;
int src = 0;
if(nPatchPerProc > 0)
{
// MPI_Request reqs[5];
MPI_Status status[5];
Box bxdst, bxsrc;
RectMDArray<double>* phiex = new RectMDArray<double>();
RectMDArray<double>* lphca = new RectMDArray<double>();
RectMDArray<double>* lphex = new RectMDArray<double>();
RectMDArray<double>* error = new RectMDArray<double>();
int idx;
for(idx = 0; idx < nPatchPerProc; idx++)
{
// Box bxdst=((const BoxLayout&) layout)[*blit];
patchID++;
// MPI_Recv(phiex, sizeof(RectMDArray<double>), MPI_CHAR, src, 0, MPI_COMM_WORLD, &status[0]);
// MPI_Recv(lphca, sizeof(RectMDArray<double>), MPI_CHAR, src, 1, MPI_COMM_WORLD, &status[1]);
// double *sourceDataPointer = (double*) malloc(sizeof(double)*phiex->getBox().sizeOf());
MPI_Recv(&bxsrc, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[0]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
MPI_Recv(&bxdst, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[1]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
phiex->define(bxsrc);
lphca->define(bxdst);
lphex->define(bxdst);
error->define(bxdst);
MPI_Recv(phiex -> getPointer(), phiex->getBox().sizeOf(), MPI_DOUBLE, src, 2, MPI_COMM_WORLD, &status[2]);
cout << "Rank " << rank << " receive patch " << idx << " sourceDataPointer with size " << phiex->getBox().sizeOf() << endl;
// double *destinationDataPointer = (double*) malloc(sizeof(double)*lphca->getBox().sizeOf());
MPI_Recv(lphca -> getPointer(), lphca->getBox().sizeOf(), MPI_DOUBLE, src, 3, MPI_COMM_WORLD, &status[3]);
cout << "Rank " << rank << " receive patch " << idx << " destinationDataPointer with size " << lphca->getBox().sizeOf() << endl;
cout << "Rank " << rank << " is working on patch " << npatches << endl;
double tmp;
tmp = doWork(bxdst, *phiex, *lphca, *lphex, *error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
}
}
MPI_Allreduce(&local_maxError, &a_maxError, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if(rank == 0)
cout << "max Error is: " << a_maxError << endl;
}