void init() { /* copy singleton simulationBox data to simbox simBox holds global and* * local SimulationSize and where the local SimArea is in the greater * * scheme using Offsets from global LEFT, TOP, FRONT */ PMACC_AUTO(simBox, Environment<DIM2>::get().SubGrid().getSimulationBox()); /* Recall that in types.hpp the following is defined: * * typedef MappingDescription<DIM2, math::CT::Int<16,16> > MappingDesc; * * where math::CT::Int<16,16> is arbitrarily(!) chosen SuperCellSize and DIM2 * * is the dimension of the grid. * * Expression of 2nd argument translates to DataSpace<DIM3>(16,16,0). * * This is the guard size (here set to be one Supercell wide in all * * directions). Meaning we have 16*16*(2*grid.x+2*grid.y+4) more * * cells in GridLayout than in SimulationBox. */ GridLayout<DIM2> layout( simBox.getLocalSize(), MappingDesc::SuperCellSize::toRT()); /* getDataSpace will return DataSpace( grid.x +16+16, grid.y +16+16) * * init stores the arguments internally in a MappingDesc private * * variable which stores the layout regarding Core, Border and guard * * in units of SuperCells to be used by the kernel to identify itself.*/ evo.init(MappingDesc(layout.getDataSpace(), 1, 1)); buff1 = new Buffer(layout, false); buff2 = new Buffer(layout, false); Space gardingCells(1, 1); for (uint32_t i = 1; i < traits::NumberOfExchanges<DIM2>::value; ++i) { /* to check which number corresponds to which direction, you can * * use the following member of class Mask like done in the two * * lines below: * * DataSpace<DIM2>relVec = Mask::getRelativeDirections<DIM2>(i); * * std::cout << "Direction:" << i << " => Vec: (" << relVec[0] * * << "," << relVec[1] << ")\n"; * * The result is: 1:right(1,0), 2:left(-1,0), 3:up(0,1), * * 4:up right(1,1), 5:(-1,1), 6:(0,-1), 7:(1,-1), 8:(-1,-1) */ /* types.hpp: enum CommunicationTags{ BUFF1 = 0u, BUFF2 = 1u }; */ buff1->addExchange(GUARD, Mask(i), gardingCells, BUFF1); buff2->addExchange(GUARD, Mask(i), gardingCells, BUFF2); } /* Both next lines are defined in GatherSlice.hpp: * * -gather saves the MessageHeader object * * -Then do an Allgather for the gloabalRanks from GC, sort out * * -inactive processes (second/boolean ,argument in gather.init) and* * save new MPI_COMMUNICATOR created from these into private var. * * -return if rank == 0 */ MessageHeader header(gridSize, layout, simBox.getGlobalOffset()); isMaster = gather.init(header, true); /* Calls kernel to initialize random generator. Game of Life is then * * initialized using uniform random numbers. With 10% (second arg) * * white points. World will be written to buffer in first argument */ evo.initEvolution(buff1->getDeviceBuffer().getDataBox(), 0.1); }
void finalize() { gather.finalize(); __delete(buff1); __delete(buff2); }
void init() { /* subGrid holds global and * local SimulationSize and where the local SimArea is in the greater * scheme using Offsets from global LEFT, TOP, FRONT */ const SubGrid<DIM2>& subGrid = Environment<DIM2>::get().SubGrid(); /* The following sets up the local layout which consists of the actual * grid cells and some surrounding cells, called guards. * * ASCII Visualization: example taken for 1D, * distributed over 2 GPUs, only 1 border shown between those two GPUs * assuming non-periodic boundary conditions. * In a N-GPU or periodic example, border cells guard cells exist in each direction. * _______GPU 0________ _______GPU 1________ * | 0 | 1 | 2 | 3 | 4 | | 3 | 4 | 5 | 6 | 7 | <-- Global (super)cell idx * |___|___|___|___|___| |___|___|___|___|___| * |___Core____|Bor|Gua| |Gua|Bor|___Core____| * |___________|der|rd_| |rd_|der|___________| * |__"real" cells_|***| |***|__"real" cells_| * * |***| Clones cells which correspond to the border cells of the neighbor GPU * (sometimes also called "ghost" or "halo" cells/region) * * Recall that the following is defined: * typedef MappingDescription<DIM2, math::CT::Int<16,16> > MappingDesc; * where math::CT::Int<16,16> is arbitrarily(!) chosen SuperCellSize * and DIM2 is the dimension of the grid. * Expression of 2nd argument translates to DataSpace<DIM3>(16,16,0). * This is the guard size (here set to be one Supercell wide in all * directions). Meaning we have 16*16*(2*grid.x+2*grid.y+4) more * cells in GridLayout than in the SubGrid. * The formula above is SuperCellSize * TotalNumGuardCells with (in this case) * SuperCellSize = 16*16 (16 cells in 2 dimensions) * TotalNumGuardCells = 2 * grid.x (top and bottom) * + 2 * grid.y (left and right) * + 4 (the corners) */ GridLayout<DIM2> layout( subGrid.getLocalDomain().size, MappingDesc::SuperCellSize::toRT()); /* getDataSpace will return DataSpace( grid.x +16+16, grid.y +16+16) * * MappingDesc stores the layout regarding Core, Border and Guard * * in units of SuperCells. * * This is saved by init to be used by the kernel to identify itself. */ evo.init(layout.getDataSpace(), Space::create(1)); buff1 = new Buffer(layout, false); buff2 = new Buffer(layout, false); /* Set up the future data exchange. In this case we need to copy the * border cells of our neighbors to our guard cells, since we only read * from the guard cells but never write to it. * guardingCells holds the number of guard(super)cells in each dimension */ Space guardingCells(1, 1); for (uint32_t i = 1; i < traits::NumberOfExchanges<DIM2>::value; ++i) { /* to check which number corresponds to which direction, you can * * use the following member of class Mask like done in the two * * lines below: * * DataSpace<DIM2>relVec = Mask::getRelativeDirections<DIM2>(i); * * std::cout << "Direction:" << i << " => Vec: (" << relVec[0] * * << "," << relVec[1] << ")\n"; * * The result is: 1:right(1,0), 2:left(-1,0), 3:up(0,1), * * 4:up right(1,1), 5:(-1,1), 6:(0,-1), 7:(1,-1), 8:(-1,-1) */ /* types.hpp: enum CommunicationTags{ BUFF1 = 0u, BUFF2 = 1u }; */ buff1->addExchange(GUARD, Mask(i), guardingCells, BUFF1); buff2->addExchange(GUARD, Mask(i), guardingCells, BUFF2); } /* Both next lines are defined in GatherSlice.hpp: * * -gather saves the MessageHeader object * * -Then do an Allgather for the gloabalRanks from GC, sort out * * -inactive processes (second/boolean ,argument in gather.init) and* * save new MPI_COMMUNICATOR created from these into private var. * * -return if rank == 0 */ MessageHeader header(gridSize, layout, subGrid.getLocalDomain().offset); isMaster = gather.init(header, true); /* Calls kernel to initialize random generator. Game of Life is then * * initialized using uniform random numbers. With 10% (second arg) * * white points. World will be written to buffer in first argument */ evo.initEvolution(buff1->getDeviceBuffer().getDataBox(), 0.1); }