//checks equality of the vector of boxes inside m_boxes
bool BoxLayout::sameBoxes(const BoxLayout& a_layout) const
{
bool retval;
if (size() == a_layout.size())
{
retval = true;
for (int iBox = 0; iBox < size(); ++iBox)
{
//RefCountedPtr<Vector<Entry> > m_boxes;
if ((*m_boxes)[iBox].box != (*a_layout.m_boxes)[iBox].box)
{
retval = false;
}
}
}
else
{
retval = false;
}
return retval;
}
Real DotProduct(const BoxLayoutData<FArrayBox>& a_dataOne,
const BoxLayoutData<FArrayBox>& a_dataTwo,
const BoxLayout& a_dblIn,
const Interval& a_comps)
{
Vector<Real> rhodot;
rhodot.reserve(10);
const int startcomp = a_comps.begin();
const int endcomp = a_comps.end();
//calculate the single-processor dot product
DataIterator dit = a_dataOne.dataIterator();
for (dit.reset(); dit.ok(); ++dit)
{
Box fabbox = a_dblIn.get(dit());
const FArrayBox& onefab = a_dataOne[dit()];
const FArrayBox& twofab = a_dataTwo[dit()];
CH_assert(onefab.box().contains(fabbox));
CH_assert(twofab.box().contains(fabbox));
Real dotgrid = 0;
FORT_DOTPRODUCT(CHF_REAL(dotgrid),
CHF_CONST_FRA(onefab),
CHF_CONST_FRA(twofab),
CHF_BOX(fabbox),
CHF_CONST_INT(startcomp),
CHF_CONST_INT(endcomp));
rhodot.push_back(dotgrid);
}
// now for the multi-processor fandango
//gather all the rhodots onto a vector and add them up
int baseProc = 0;
Vector<Vector<Real> >dotVec;
gather(dotVec, rhodot, baseProc);
Real rhodotTot = 0.0;
if (procID() == baseProc)
{
CH_assert(dotVec.size() == numProc());
rhodot.resize(a_dblIn.size());
int index = 0;
for (int p = 0; p < dotVec.size(); p++)
{
Vector<Real>& v = dotVec[p];
for (int i = 0; i < v.size(); i++, index++)
{
rhodot[index] = v[i];
}
}
rhodot.sort();
for (int ivec = 0; ivec < rhodot.size(); ivec++)
{
rhodotTot += rhodot[ivec];
}
}
//broadcast the sum to all processors.
broadcast(rhodotTot, baseProc);
//return the total
return rhodotTot;
}
void getError(LevelData<double, 1> & a_error,
double & a_maxError,
const double & a_dx)
{
int rank, nprocs;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
MPI_Comm_size (MPI_COMM_WORLD, &nprocs);
std::cout << "I am rank " << rank << " of " << nprocs << " processes" << std::endl;
BoxLayout* layout = new BoxLayout();
int npatches;
int nPatchPerProc, iStartIdx, iEndIdx;
int patchID = -1;
double local_maxError = 0;
LevelData<double, 1> *phi;
LevelData<double, 1> *lphcalc;
LevelData<double, 1> *lphexac;
if(rank == 0)
{
*layout = a_error.getBoxLayout();
npatches = layout->size();
phi = new LevelData<double, 1>(*layout, s_nghost);
lphcalc = new LevelData<double, 1>(*layout, 0);
lphexac = new LevelData<double, 1>(*layout, 0);
// cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl;
initialize(*phi,*lphexac, a_dx);
//set ghost cells of phi
phi->exchange();
}
if(nprocs > 1)
MPI_Bcast( &npatches, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
if(npatches == 1)
nPatchPerProc = 1;
else
nPatchPerProc = npatches/nprocs;
iStartIdx = rank * nPatchPerProc;
iEndIdx = (npatches < (rank+1)*nPatchPerProc) ? npatches-1 : ((rank+1)*nPatchPerProc-1);
nPatchPerProc = iEndIdx - iStartIdx + 1;
std::cout << "I am rank " << rank << " working from " << iStartIdx << " to " << iEndIdx << std::endl;
// barrier to sync halo exchange
if(rank == 0)
{
Box bxdst, bxsrc;
MPI_Request reqs[5];
MPI_Status status[5];
int iwait = 0;
for(BLIterator blit(*layout); blit != blit.end(); ++blit)
{
//bxdst= (*layout)[*blit];
patchID++;
bool mypatch = patchID >= iStartIdx && patchID <=iEndIdx;
RectMDArray<double>& phiex = (*phi)[patchID];
RectMDArray<double>& lphca = (*lphcalc)[patchID];
RectMDArray<double>& lphex = (*lphexac)[patchID];
RectMDArray<double>& error = a_error[patchID];
bxsrc = phiex . getBox();
bxdst = lphca . getBox();
lphca.setVal(0.);
if(mypatch)
{
cout << "Rank 0 is working on patch " << patchID << endl;
double tmp;
tmp = doWork(bxdst, phiex, lphca, lphex, error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
else
{
int dest = patchID / nPatchPerProc;
// MPI_Isend(&phiex, sizeof(RectMDArray<double>),MPI_CHAR, dest, 0, MPI_COMM_WORLD,&reqs[0]);
//cout << "master send out patch " << patchID<< " sourceDataPointer with size " << phiex.getBox().sizeOf() << endl;
// MPI_Isend(&lphca, sizeof(RectMDArray<double>),MPI_CHAR, dest, 1, MPI_COMM_WORLD,&reqs[1]);
//cout << "master send out patch " << patchID<< " destinationDataPointer with size " << lphca.getBox().sizeOf() << endl;
MPI_Isend(&bxsrc, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[0]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
MPI_Isend(&bxdst, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[1]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
double *sourceDataPointer = phiex . getPointer();
double *destinationDataPointer = lphca . getPointer();
MPI_Isend(sourceDataPointer, phiex.getBox().sizeOf(),MPI_DOUBLE, dest, 2, MPI_COMM_WORLD,&reqs[2]);
MPI_Isend(destinationDataPointer, lphca.getBox().sizeOf(),MPI_DOUBLE, dest, 3, MPI_COMM_WORLD,&reqs[3]);
iwait++;
MPI_Waitall(4,reqs,status);
cout << "Rank 0 is sending patch " << patchID << " to rank " << dest << endl;
}
}
}
else
{
if(npatches == 1)
nPatchPerProc = 0;
else
nPatchPerProc = npatches/nprocs;
int src = 0;
if(nPatchPerProc > 0)
{
// MPI_Request reqs[5];
MPI_Status status[5];
Box bxdst, bxsrc;
RectMDArray<double>* phiex = new RectMDArray<double>();
RectMDArray<double>* lphca = new RectMDArray<double>();
RectMDArray<double>* lphex = new RectMDArray<double>();
RectMDArray<double>* error = new RectMDArray<double>();
int idx;
for(idx = 0; idx < nPatchPerProc; idx++)
{
// Box bxdst=((const BoxLayout&) layout)[*blit];
patchID++;
// MPI_Recv(phiex, sizeof(RectMDArray<double>), MPI_CHAR, src, 0, MPI_COMM_WORLD, &status[0]);
// MPI_Recv(lphca, sizeof(RectMDArray<double>), MPI_CHAR, src, 1, MPI_COMM_WORLD, &status[1]);
// double *sourceDataPointer = (double*) malloc(sizeof(double)*phiex->getBox().sizeOf());
MPI_Recv(&bxsrc, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[0]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
MPI_Recv(&bxdst, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[1]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
phiex->define(bxsrc);
lphca->define(bxdst);
lphex->define(bxdst);
error->define(bxdst);
MPI_Recv(phiex -> getPointer(), phiex->getBox().sizeOf(), MPI_DOUBLE, src, 2, MPI_COMM_WORLD, &status[2]);
cout << "Rank " << rank << " receive patch " << idx << " sourceDataPointer with size " << phiex->getBox().sizeOf() << endl;
// double *destinationDataPointer = (double*) malloc(sizeof(double)*lphca->getBox().sizeOf());
MPI_Recv(lphca -> getPointer(), lphca->getBox().sizeOf(), MPI_DOUBLE, src, 3, MPI_COMM_WORLD, &status[3]);
cout << "Rank " << rank << " receive patch " << idx << " destinationDataPointer with size " << lphca->getBox().sizeOf() << endl;
cout << "Rank " << rank << " is working on patch " << npatches << endl;
double tmp;
tmp = doWork(bxdst, *phiex, *lphca, *lphex, *error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
}
}
MPI_Allreduce(&local_maxError, &a_maxError, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if(rank == 0)
cout << "max Error is: " << a_maxError << endl;
}