void
Multigrid::fineInterp(
LevelData<double >& a_phi,
const LevelData<double >& a_delta
)
{
CH_TIMERS("Multigrid::fineInterp");
BLIterator blit(m_bl);
BoxLayout blCoarse = a_delta.getBoxLayout();
Box bxKernel(getZeros(),getOnes());
for (blit.begin();blit != blit.end();++blit)
{
Box bx0 = blCoarse[*blit];
const RectMDArray<double >& delta = a_delta[*blit];
Box bxc = a_delta.getBoxLayout()[*blit];
RectMDArray<double >& phi = a_phi[*blit];
for (Point ptsten = getZeros();
bxKernel.notDone(ptsten);
bxKernel.increment(ptsten))
{
g_FineInterp.setDestShift(ptsten);
phi += g_FineInterp(delta,bx0);
}
}
};
void initialize(LevelData<double, 1> & a_phi,
LevelData<double, 1> & a_lphExac,
const double & a_dx)
{
const BoxLayout& layout = a_phi.getBoxLayout();
for(BLIterator blit(layout); blit != blit.end(); ++blit)
{
Box bx = layout[*blit];
initialize(a_phi [*blit],
a_lphExac[*blit],
a_dx, bx
);
}
}
void initialize(LevelData<double >& a_phi,double a_dx)
{
BoxLayout bl = a_phi.getBoxLayout();
for (BLIterator blit(bl); blit != blit.end(); ++blit)
{
RectMDArray<double >& phi = a_phi[*blit];
Box bx = bl[*blit];
for (Point pt = bx.getLowCorner(); bx.notDone(pt); bx.increment(pt))
{
phi[pt] = 1.;
for (int idir = 0; idir < DIM ; idir++)
{
phi[pt] = phi[pt]*sin((M_PI*2*pt[idir])*a_dx);
}
}
}
};
void getError(LevelData<double, 1> & a_error,
double & a_maxError,
const double & a_dx)
{
BoxLayout layout = a_error.getBoxLayout();
LevelData<double, 1> phi(layout, s_nghost);
LevelData<double, 1> lphcalc(layout, 0);
LevelData<double, 1> lphexac(layout, 0);
// cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl;
initialize(phi,lphexac, a_dx);
//set ghost cells of phi
phi.exchange();
// dumpLevelRDA(&phi);
Stencil<double> laplace;
setStencil(laplace, a_dx);
//apply stencil operator independently on each box
a_maxError = 0;
for(BLIterator blit(layout); blit != blit.end(); ++blit)
{
RectMDArray<double>& phiex = phi[*blit];
RectMDArray<double>& lphca = lphcalc[*blit];
RectMDArray<double>& lphex = lphexac[*blit];
RectMDArray<double>& error = a_error[*blit];
//apply is set as an increment so need to set this to zero initially
lphca.setVal(0.);
Box bxdst=layout[*blit];
Stencil<double>::apply(laplace, phiex, lphca, bxdst);
//error = lphicalc -lphiexac
lphca.copyTo(error);
//here err holds lphi calc
double maxbox = forall_max(error, lphex, &errorF, bxdst);
a_maxError = max(maxbox, a_maxError);
}
}
void testlap(LevelData<double >& a_phi, double a_dx,char* a_str)
{
double coef = 1./(a_dx*a_dx);
Stencil<double> Laplacian(make_pair(getZeros(),-DIM*2*coef));
BoxLayout bl = a_phi.getBoxLayout();
for (int dir = 0; dir < DIM ; dir++)
{
Point edir = getUnitv(dir);
Stencil<double> plus(make_pair(Shift(edir),coef));
Stencil<double> minus(make_pair(Shift(edir*(-1)),coef));
Laplacian = Laplacian + minus + plus;
}
a_phi.exchange();
RectMDArray<double> LPhi00(bl.getDomain());
for (BLIterator blit(bl); blit != blit.end(); ++blit)
{
LPhi00 |= Laplacian(a_phi[*blit],bl[*blit]);
}
MDWrite(a_str,LPhi00);
};
void
Multigrid::avgDown(
LevelData<double >& a_resc,
const LevelData<double >& a_res
)
{
CH_TIMERS("Multigrid::avgDown");
// Conservative, finite-volume averaging from coarse to fine.
BLIterator blit(m_bl);
for (blit.begin();blit !=blit.end();++blit)
{
RectMDArray<double >& resc = a_resc[*blit];
resc.setVal(0.0);
Box bxc = a_resc.getBoxLayout()[*blit];
const RectMDArray<double >& res = a_res[*blit];
for (int i = 0; i < (1 << DIM); i++)
{
resc += g_AvgDown[i](res,bxc);
}
}
};
void getError(LevelData<double, 1> & a_error,
double & a_maxError,
const double & a_dx)
{
BoxLayout layout = a_error.getBoxLayout();
LevelData<double, 1> phi(layout, s_nghost);
LevelData<double, 1> lphcalc(layout, 0);
LevelData<double, 1> lphexac(layout, 0);
// cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl;
initialize(phi,lphexac, a_dx);
//set ghost cells of phi
phi.exchange();
// dumpLevelRDA(&phi);
Stencil<double> laplace;
setStencil(laplace, a_dx);
//apply stencil operator independently on each box
a_maxError = 0;
for(BLIterator blit(layout); blit != blit.end(); ++blit)
{
RectMDArray<double>& phiex = phi[*blit];
RectMDArray<double>& lphca = lphcalc[*blit];
RectMDArray<double>& lphex = lphexac[*blit];
RectMDArray<double>& error = a_error[*blit];
//apply is set as an increment so need to set this to zero initially
lphca.setVal(0.);
Box bxdst=layout[*blit];
// Stencil<double>::apply(laplace, phiex, lphca, bxdst);
const class Box sourceBoxRef = phiex . getBox();
const class Box destinationBoxRef = lphca . getBox();
int iter_lb2 = bxdst . getLowCorner ()[2];
int src_lb2 = sourceBoxRef . getLowCorner ()[2];
int dest_lb2 = destinationBoxRef . getLowCorner ()[2];
int k = 0;
int iter_ub2 = bxdst . getHighCorner ()[2];
int src_ub2 = sourceBoxRef . getHighCorner ()[2];
int dest_ub2 = destinationBoxRef . getHighCorner ()[2];
int arraySize_X = bxdst . size (0);
int arraySize_X_src = sourceBoxRef . size (0);
int iter_lb1 = bxdst . getLowCorner ()[1];
int src_lb1 = sourceBoxRef . getLowCorner ()[1];
int dest_lb1 = destinationBoxRef . getLowCorner ()[1];
int j = 0;
int iter_ub1 = bxdst . getHighCorner ()[1];
int src_ub1 = sourceBoxRef . getHighCorner ()[1];
int dest_ub1 = destinationBoxRef . getHighCorner ()[1];
int arraySize_Y = bxdst . size (1);
int arraySize_Y_src = sourceBoxRef . size (1);
int iter_lb0 = bxdst . getLowCorner ()[0];
int src_lb0 = sourceBoxRef . getLowCorner ()[0];
int dest_lb0 = destinationBoxRef . getLowCorner ()[0];
int i = 0;
int iter_ub0 = bxdst . getHighCorner ()[0];
int src_ub0 = sourceBoxRef . getHighCorner ()[0];
int dest_ub0 = destinationBoxRef . getHighCorner ()[0];
int arraySize_Z = bxdst . size (2);
int arraySize_Z_src = sourceBoxRef . size (2);
double *sourceDataPointer = phiex . getPointer();
double *destinationDataPointer = lphca . getPointer();
for (k = iter_lb2; k < iter_ub2; ++k) {
for (j = iter_lb1; j < iter_ub1; ++j) {
for (i = iter_lb0; i < iter_ub0; ++i) {
destinationDataPointer[arraySize_X * (arraySize_Y * (k - dest_lb2) + (j - dest_lb1)) + (i - dest_lb0)] =
(1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2 + -1) + (j - src_lb1)) + (i - src_lb0)] +
1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2 + 1) + (j - src_lb1)) + (i - src_lb0)] +
1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1 + -1)) + (i - src_lb0)] +
1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1 + 1)) + (i - src_lb0)] +
1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1)) + (i - src_lb0 + -1)] +
1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1)) + (i - src_lb0 + 1)] +
(-2.0*DIM)*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1)) + (i - src_lb0)]) * (1.0/a_dx/a_dx);
//cout << destinationDataPointer[arraySize_X * (arraySize_Y * (k - dest_lb2) + (j - dest_lb1)) + (i - dest_lb0)] << " " << lphex.getPointer()[arraySize_X * (arraySize_Y * (k - dest_lb2) + (j - dest_lb1)) + (i - dest_lb0)] << endl;
}
}
}
//error = lphicalc -lphiexac
lphca.copyTo(error);
//here err holds lphi calc
double maxbox = forall_max(error, lphex, &errorF, bxdst);
cout << "maxbox= " << maxbox << endl;
a_maxError = max(maxbox, a_maxError);
}
}
void getError(LevelData<double, 1> & a_error,
double & a_maxError,
const double & a_dx)
{
int rank, nprocs;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
MPI_Comm_size (MPI_COMM_WORLD, &nprocs);
std::cout << "I am rank " << rank << " of " << nprocs << " processes" << std::endl;
BoxLayout* layout = new BoxLayout();
int npatches;
int nPatchPerProc, iStartIdx, iEndIdx;
int patchID = -1;
double local_maxError = 0;
LevelData<double, 1> *phi;
LevelData<double, 1> *lphcalc;
LevelData<double, 1> *lphexac;
if(rank == 0)
{
*layout = a_error.getBoxLayout();
npatches = layout->size();
phi = new LevelData<double, 1>(*layout, s_nghost);
lphcalc = new LevelData<double, 1>(*layout, 0);
lphexac = new LevelData<double, 1>(*layout, 0);
// cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl;
initialize(*phi,*lphexac, a_dx);
//set ghost cells of phi
phi->exchange();
}
if(nprocs > 1)
MPI_Bcast( &npatches, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
if(npatches == 1)
nPatchPerProc = 1;
else
nPatchPerProc = npatches/nprocs;
iStartIdx = rank * nPatchPerProc;
iEndIdx = (npatches < (rank+1)*nPatchPerProc) ? npatches-1 : ((rank+1)*nPatchPerProc-1);
nPatchPerProc = iEndIdx - iStartIdx + 1;
std::cout << "I am rank " << rank << " working from " << iStartIdx << " to " << iEndIdx << std::endl;
// barrier to sync halo exchange
if(rank == 0)
{
Box bxdst, bxsrc;
MPI_Request reqs[5];
MPI_Status status[5];
int iwait = 0;
for(BLIterator blit(*layout); blit != blit.end(); ++blit)
{
//bxdst= (*layout)[*blit];
patchID++;
bool mypatch = patchID >= iStartIdx && patchID <=iEndIdx;
RectMDArray<double>& phiex = (*phi)[patchID];
RectMDArray<double>& lphca = (*lphcalc)[patchID];
RectMDArray<double>& lphex = (*lphexac)[patchID];
RectMDArray<double>& error = a_error[patchID];
bxsrc = phiex . getBox();
bxdst = lphca . getBox();
lphca.setVal(0.);
if(mypatch)
{
cout << "Rank 0 is working on patch " << patchID << endl;
double tmp;
tmp = doWork(bxdst, phiex, lphca, lphex, error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
else
{
int dest = patchID / nPatchPerProc;
// MPI_Isend(&phiex, sizeof(RectMDArray<double>),MPI_CHAR, dest, 0, MPI_COMM_WORLD,&reqs[0]);
//cout << "master send out patch " << patchID<< " sourceDataPointer with size " << phiex.getBox().sizeOf() << endl;
// MPI_Isend(&lphca, sizeof(RectMDArray<double>),MPI_CHAR, dest, 1, MPI_COMM_WORLD,&reqs[1]);
//cout << "master send out patch " << patchID<< " destinationDataPointer with size " << lphca.getBox().sizeOf() << endl;
MPI_Isend(&bxsrc, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[0]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
MPI_Isend(&bxdst, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[1]);
cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl;
double *sourceDataPointer = phiex . getPointer();
double *destinationDataPointer = lphca . getPointer();
MPI_Isend(sourceDataPointer, phiex.getBox().sizeOf(),MPI_DOUBLE, dest, 2, MPI_COMM_WORLD,&reqs[2]);
MPI_Isend(destinationDataPointer, lphca.getBox().sizeOf(),MPI_DOUBLE, dest, 3, MPI_COMM_WORLD,&reqs[3]);
iwait++;
MPI_Waitall(4,reqs,status);
cout << "Rank 0 is sending patch " << patchID << " to rank " << dest << endl;
}
}
}
else
{
if(npatches == 1)
nPatchPerProc = 0;
else
nPatchPerProc = npatches/nprocs;
int src = 0;
if(nPatchPerProc > 0)
{
// MPI_Request reqs[5];
MPI_Status status[5];
Box bxdst, bxsrc;
RectMDArray<double>* phiex = new RectMDArray<double>();
RectMDArray<double>* lphca = new RectMDArray<double>();
RectMDArray<double>* lphex = new RectMDArray<double>();
RectMDArray<double>* error = new RectMDArray<double>();
int idx;
for(idx = 0; idx < nPatchPerProc; idx++)
{
// Box bxdst=((const BoxLayout&) layout)[*blit];
patchID++;
// MPI_Recv(phiex, sizeof(RectMDArray<double>), MPI_CHAR, src, 0, MPI_COMM_WORLD, &status[0]);
// MPI_Recv(lphca, sizeof(RectMDArray<double>), MPI_CHAR, src, 1, MPI_COMM_WORLD, &status[1]);
// double *sourceDataPointer = (double*) malloc(sizeof(double)*phiex->getBox().sizeOf());
MPI_Recv(&bxsrc, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[0]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
MPI_Recv(&bxdst, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[1]);
cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl;
phiex->define(bxsrc);
lphca->define(bxdst);
lphex->define(bxdst);
error->define(bxdst);
MPI_Recv(phiex -> getPointer(), phiex->getBox().sizeOf(), MPI_DOUBLE, src, 2, MPI_COMM_WORLD, &status[2]);
cout << "Rank " << rank << " receive patch " << idx << " sourceDataPointer with size " << phiex->getBox().sizeOf() << endl;
// double *destinationDataPointer = (double*) malloc(sizeof(double)*lphca->getBox().sizeOf());
MPI_Recv(lphca -> getPointer(), lphca->getBox().sizeOf(), MPI_DOUBLE, src, 3, MPI_COMM_WORLD, &status[3]);
cout << "Rank " << rank << " receive patch " << idx << " destinationDataPointer with size " << lphca->getBox().sizeOf() << endl;
cout << "Rank " << rank << " is working on patch " << npatches << endl;
double tmp;
tmp = doWork(bxdst, *phiex, *lphca, *lphex, *error, a_dx);
local_maxError = (local_maxError > tmp) ? local_maxError : tmp;
}
}
}
MPI_Allreduce(&local_maxError, &a_maxError, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if(rank == 0)
cout << "max Error is: " << a_maxError << endl;
}