void Multigrid::fineInterp( LevelData<double >& a_phi, const LevelData<double >& a_delta ) { CH_TIMERS("Multigrid::fineInterp"); BLIterator blit(m_bl); BoxLayout blCoarse = a_delta.getBoxLayout(); Box bxKernel(getZeros(),getOnes()); for (blit.begin();blit != blit.end();++blit) { Box bx0 = blCoarse[*blit]; const RectMDArray<double >& delta = a_delta[*blit]; Box bxc = a_delta.getBoxLayout()[*blit]; RectMDArray<double >& phi = a_phi[*blit]; for (Point ptsten = getZeros(); bxKernel.notDone(ptsten); bxKernel.increment(ptsten)) { g_FineInterp.setDestShift(ptsten); phi += g_FineInterp(delta,bx0); } } };
void initialize(LevelData<double, 1> & a_phi, LevelData<double, 1> & a_lphExac, const double & a_dx) { const BoxLayout& layout = a_phi.getBoxLayout(); for(BLIterator blit(layout); blit != blit.end(); ++blit) { Box bx = layout[*blit]; initialize(a_phi [*blit], a_lphExac[*blit], a_dx, bx ); } }
void initialize(LevelData<double >& a_phi,double a_dx) { BoxLayout bl = a_phi.getBoxLayout(); for (BLIterator blit(bl); blit != blit.end(); ++blit) { RectMDArray<double >& phi = a_phi[*blit]; Box bx = bl[*blit]; for (Point pt = bx.getLowCorner(); bx.notDone(pt); bx.increment(pt)) { phi[pt] = 1.; for (int idir = 0; idir < DIM ; idir++) { phi[pt] = phi[pt]*sin((M_PI*2*pt[idir])*a_dx); } } } };
void getError(LevelData<double, 1> & a_error, double & a_maxError, const double & a_dx) { BoxLayout layout = a_error.getBoxLayout(); LevelData<double, 1> phi(layout, s_nghost); LevelData<double, 1> lphcalc(layout, 0); LevelData<double, 1> lphexac(layout, 0); // cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl; initialize(phi,lphexac, a_dx); //set ghost cells of phi phi.exchange(); // dumpLevelRDA(&phi); Stencil<double> laplace; setStencil(laplace, a_dx); //apply stencil operator independently on each box a_maxError = 0; for(BLIterator blit(layout); blit != blit.end(); ++blit) { RectMDArray<double>& phiex = phi[*blit]; RectMDArray<double>& lphca = lphcalc[*blit]; RectMDArray<double>& lphex = lphexac[*blit]; RectMDArray<double>& error = a_error[*blit]; //apply is set as an increment so need to set this to zero initially lphca.setVal(0.); Box bxdst=layout[*blit]; Stencil<double>::apply(laplace, phiex, lphca, bxdst); //error = lphicalc -lphiexac lphca.copyTo(error); //here err holds lphi calc double maxbox = forall_max(error, lphex, &errorF, bxdst); a_maxError = max(maxbox, a_maxError); } }
void testlap(LevelData<double >& a_phi, double a_dx,char* a_str) { double coef = 1./(a_dx*a_dx); Stencil<double> Laplacian(make_pair(getZeros(),-DIM*2*coef)); BoxLayout bl = a_phi.getBoxLayout(); for (int dir = 0; dir < DIM ; dir++) { Point edir = getUnitv(dir); Stencil<double> plus(make_pair(Shift(edir),coef)); Stencil<double> minus(make_pair(Shift(edir*(-1)),coef)); Laplacian = Laplacian + minus + plus; } a_phi.exchange(); RectMDArray<double> LPhi00(bl.getDomain()); for (BLIterator blit(bl); blit != blit.end(); ++blit) { LPhi00 |= Laplacian(a_phi[*blit],bl[*blit]); } MDWrite(a_str,LPhi00); };
void Multigrid::avgDown( LevelData<double >& a_resc, const LevelData<double >& a_res ) { CH_TIMERS("Multigrid::avgDown"); // Conservative, finite-volume averaging from coarse to fine. BLIterator blit(m_bl); for (blit.begin();blit !=blit.end();++blit) { RectMDArray<double >& resc = a_resc[*blit]; resc.setVal(0.0); Box bxc = a_resc.getBoxLayout()[*blit]; const RectMDArray<double >& res = a_res[*blit]; for (int i = 0; i < (1 << DIM); i++) { resc += g_AvgDown[i](res,bxc); } } };
void getError(LevelData<double, 1> & a_error, double & a_maxError, const double & a_dx) { BoxLayout layout = a_error.getBoxLayout(); LevelData<double, 1> phi(layout, s_nghost); LevelData<double, 1> lphcalc(layout, 0); LevelData<double, 1> lphexac(layout, 0); // cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl; initialize(phi,lphexac, a_dx); //set ghost cells of phi phi.exchange(); // dumpLevelRDA(&phi); Stencil<double> laplace; setStencil(laplace, a_dx); //apply stencil operator independently on each box a_maxError = 0; for(BLIterator blit(layout); blit != blit.end(); ++blit) { RectMDArray<double>& phiex = phi[*blit]; RectMDArray<double>& lphca = lphcalc[*blit]; RectMDArray<double>& lphex = lphexac[*blit]; RectMDArray<double>& error = a_error[*blit]; //apply is set as an increment so need to set this to zero initially lphca.setVal(0.); Box bxdst=layout[*blit]; // Stencil<double>::apply(laplace, phiex, lphca, bxdst); const class Box sourceBoxRef = phiex . getBox(); const class Box destinationBoxRef = lphca . getBox(); int iter_lb2 = bxdst . getLowCorner ()[2]; int src_lb2 = sourceBoxRef . getLowCorner ()[2]; int dest_lb2 = destinationBoxRef . getLowCorner ()[2]; int k = 0; int iter_ub2 = bxdst . getHighCorner ()[2]; int src_ub2 = sourceBoxRef . getHighCorner ()[2]; int dest_ub2 = destinationBoxRef . getHighCorner ()[2]; int arraySize_X = bxdst . size (0); int arraySize_X_src = sourceBoxRef . size (0); int iter_lb1 = bxdst . getLowCorner ()[1]; int src_lb1 = sourceBoxRef . getLowCorner ()[1]; int dest_lb1 = destinationBoxRef . getLowCorner ()[1]; int j = 0; int iter_ub1 = bxdst . getHighCorner ()[1]; int src_ub1 = sourceBoxRef . getHighCorner ()[1]; int dest_ub1 = destinationBoxRef . getHighCorner ()[1]; int arraySize_Y = bxdst . size (1); int arraySize_Y_src = sourceBoxRef . size (1); int iter_lb0 = bxdst . getLowCorner ()[0]; int src_lb0 = sourceBoxRef . getLowCorner ()[0]; int dest_lb0 = destinationBoxRef . getLowCorner ()[0]; int i = 0; int iter_ub0 = bxdst . getHighCorner ()[0]; int src_ub0 = sourceBoxRef . getHighCorner ()[0]; int dest_ub0 = destinationBoxRef . getHighCorner ()[0]; int arraySize_Z = bxdst . size (2); int arraySize_Z_src = sourceBoxRef . size (2); double *sourceDataPointer = phiex . getPointer(); double *destinationDataPointer = lphca . getPointer(); for (k = iter_lb2; k < iter_ub2; ++k) { for (j = iter_lb1; j < iter_ub1; ++j) { for (i = iter_lb0; i < iter_ub0; ++i) { destinationDataPointer[arraySize_X * (arraySize_Y * (k - dest_lb2) + (j - dest_lb1)) + (i - dest_lb0)] = (1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2 + -1) + (j - src_lb1)) + (i - src_lb0)] + 1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2 + 1) + (j - src_lb1)) + (i - src_lb0)] + 1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1 + -1)) + (i - src_lb0)] + 1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1 + 1)) + (i - src_lb0)] + 1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1)) + (i - src_lb0 + -1)] + 1.0*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1)) + (i - src_lb0 + 1)] + (-2.0*DIM)*sourceDataPointer[arraySize_X_src * (arraySize_Y_src * (k - src_lb2) + (j - src_lb1)) + (i - src_lb0)]) * (1.0/a_dx/a_dx); //cout << destinationDataPointer[arraySize_X * (arraySize_Y * (k - dest_lb2) + (j - dest_lb1)) + (i - dest_lb0)] << " " << lphex.getPointer()[arraySize_X * (arraySize_Y * (k - dest_lb2) + (j - dest_lb1)) + (i - dest_lb0)] << endl; } } } //error = lphicalc -lphiexac lphca.copyTo(error); //here err holds lphi calc double maxbox = forall_max(error, lphex, &errorF, bxdst); cout << "maxbox= " << maxbox << endl; a_maxError = max(maxbox, a_maxError); } }
void getError(LevelData<double, 1> & a_error, double & a_maxError, const double & a_dx) { int rank, nprocs; MPI_Comm_rank (MPI_COMM_WORLD, &rank); MPI_Comm_size (MPI_COMM_WORLD, &nprocs); std::cout << "I am rank " << rank << " of " << nprocs << " processes" << std::endl; BoxLayout* layout = new BoxLayout(); int npatches; int nPatchPerProc, iStartIdx, iEndIdx; int patchID = -1; double local_maxError = 0; LevelData<double, 1> *phi; LevelData<double, 1> *lphcalc; LevelData<double, 1> *lphexac; if(rank == 0) { *layout = a_error.getBoxLayout(); npatches = layout->size(); phi = new LevelData<double, 1>(*layout, s_nghost); lphcalc = new LevelData<double, 1>(*layout, 0); lphexac = new LevelData<double, 1>(*layout, 0); // cout << "initializing phi to sum_dir(sin 2*pi*xdir)" << endl; initialize(*phi,*lphexac, a_dx); //set ghost cells of phi phi->exchange(); } if(nprocs > 1) MPI_Bcast( &npatches, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD); if(npatches == 1) nPatchPerProc = 1; else nPatchPerProc = npatches/nprocs; iStartIdx = rank * nPatchPerProc; iEndIdx = (npatches < (rank+1)*nPatchPerProc) ? npatches-1 : ((rank+1)*nPatchPerProc-1); nPatchPerProc = iEndIdx - iStartIdx + 1; std::cout << "I am rank " << rank << " working from " << iStartIdx << " to " << iEndIdx << std::endl; // barrier to sync halo exchange if(rank == 0) { Box bxdst, bxsrc; MPI_Request reqs[5]; MPI_Status status[5]; int iwait = 0; for(BLIterator blit(*layout); blit != blit.end(); ++blit) { //bxdst= (*layout)[*blit]; patchID++; bool mypatch = patchID >= iStartIdx && patchID <=iEndIdx; RectMDArray<double>& phiex = (*phi)[patchID]; RectMDArray<double>& lphca = (*lphcalc)[patchID]; RectMDArray<double>& lphex = (*lphexac)[patchID]; RectMDArray<double>& error = a_error[patchID]; bxsrc = phiex . getBox(); bxdst = lphca . getBox(); lphca.setVal(0.); if(mypatch) { cout << "Rank 0 is working on patch " << patchID << endl; double tmp; tmp = doWork(bxdst, phiex, lphca, lphex, error, a_dx); local_maxError = (local_maxError > tmp) ? local_maxError : tmp; } else { int dest = patchID / nPatchPerProc; // MPI_Isend(&phiex, sizeof(RectMDArray<double>),MPI_CHAR, dest, 0, MPI_COMM_WORLD,&reqs[0]); //cout << "master send out patch " << patchID<< " sourceDataPointer with size " << phiex.getBox().sizeOf() << endl; // MPI_Isend(&lphca, sizeof(RectMDArray<double>),MPI_CHAR, dest, 1, MPI_COMM_WORLD,&reqs[1]); //cout << "master send out patch " << patchID<< " destinationDataPointer with size " << lphca.getBox().sizeOf() << endl; MPI_Isend(&bxsrc, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[0]); cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl; MPI_Isend(&bxdst, sizeof(Box),MPI_CHAR, dest, 4, MPI_COMM_WORLD,&reqs[1]); cout << "master send out patch " << patchID<< " bxdst Box with size " << sizeof(Box)<< endl; double *sourceDataPointer = phiex . getPointer(); double *destinationDataPointer = lphca . getPointer(); MPI_Isend(sourceDataPointer, phiex.getBox().sizeOf(),MPI_DOUBLE, dest, 2, MPI_COMM_WORLD,&reqs[2]); MPI_Isend(destinationDataPointer, lphca.getBox().sizeOf(),MPI_DOUBLE, dest, 3, MPI_COMM_WORLD,&reqs[3]); iwait++; MPI_Waitall(4,reqs,status); cout << "Rank 0 is sending patch " << patchID << " to rank " << dest << endl; } } } else { if(npatches == 1) nPatchPerProc = 0; else nPatchPerProc = npatches/nprocs; int src = 0; if(nPatchPerProc > 0) { // MPI_Request reqs[5]; MPI_Status status[5]; Box bxdst, bxsrc; RectMDArray<double>* phiex = new RectMDArray<double>(); RectMDArray<double>* lphca = new RectMDArray<double>(); RectMDArray<double>* lphex = new RectMDArray<double>(); RectMDArray<double>* error = new RectMDArray<double>(); int idx; for(idx = 0; idx < nPatchPerProc; idx++) { // Box bxdst=((const BoxLayout&) layout)[*blit]; patchID++; // MPI_Recv(phiex, sizeof(RectMDArray<double>), MPI_CHAR, src, 0, MPI_COMM_WORLD, &status[0]); // MPI_Recv(lphca, sizeof(RectMDArray<double>), MPI_CHAR, src, 1, MPI_COMM_WORLD, &status[1]); // double *sourceDataPointer = (double*) malloc(sizeof(double)*phiex->getBox().sizeOf()); MPI_Recv(&bxsrc, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[0]); cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl; MPI_Recv(&bxdst, sizeof(Box), MPI_CHAR, src, 4, MPI_COMM_WORLD, &status[1]); cout << "Rank " << rank << " receive patch " << idx << " bxdst with size " << sizeof(Box)<< endl; phiex->define(bxsrc); lphca->define(bxdst); lphex->define(bxdst); error->define(bxdst); MPI_Recv(phiex -> getPointer(), phiex->getBox().sizeOf(), MPI_DOUBLE, src, 2, MPI_COMM_WORLD, &status[2]); cout << "Rank " << rank << " receive patch " << idx << " sourceDataPointer with size " << phiex->getBox().sizeOf() << endl; // double *destinationDataPointer = (double*) malloc(sizeof(double)*lphca->getBox().sizeOf()); MPI_Recv(lphca -> getPointer(), lphca->getBox().sizeOf(), MPI_DOUBLE, src, 3, MPI_COMM_WORLD, &status[3]); cout << "Rank " << rank << " receive patch " << idx << " destinationDataPointer with size " << lphca->getBox().sizeOf() << endl; cout << "Rank " << rank << " is working on patch " << npatches << endl; double tmp; tmp = doWork(bxdst, *phiex, *lphca, *lphex, *error, a_dx); local_maxError = (local_maxError > tmp) ? local_maxError : tmp; } } } MPI_Allreduce(&local_maxError, &a_maxError, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); if(rank == 0) cout << "max Error is: " << a_maxError << endl; }