/**
* Main program for the simulation on a single SWE_WavePropagationBlock.
*/
int main( int argc, char** argv ) {
/**
* Initialization.
*/
// Parse command line parameters
tools::Args args;
#ifndef READXML
args.addOption("grid-size-x", 'x', "Number of cells in x direction");
args.addOption("grid-size-y", 'y', "Number of cells in y direction");
args.addOption("output-basepath", 'o', "Output base file name");
#endif
tools::Args::Result ret = args.parse(argc, argv);
switch (ret)
{
case tools::Args::Error:
return 1;
case tools::Args::Help:
return 0;
}
//! number of grid cells in x- and y-direction.
int l_nX, l_nY;
//! l_baseName of the plots.
std::string l_baseName;
// read command line parameters
#ifndef READXML
l_nX = args.getArgument<int>("grid-size-x");
l_nY = args.getArgument<int>("grid-size-y");
l_baseName = args.getArgument<std::string>("output-basepath");
#endif
// read xml file
#ifdef READXML
assert(false); //TODO: not implemented.
if(argc != 2) {
s_sweLogger.printString("Aborting. Please provide a proper input file.");
s_sweLogger.printString("Example: ./SWE_gnu_debug_none_augrie config.xml");
return 1;
}
s_sweLogger.printString("Reading xml-file.");
std::string l_xmlFile = std::string(argv[1]);
s_sweLogger.printString(l_xmlFile);
CXMLConfig l_xmlConfig;
l_xmlConfig.loadConfig(l_xmlFile.c_str());
#endif
#ifdef ASAGI
/* Information about the example bathymetry grid (tohoku_gebco_ucsb3_500m_hawaii_bath.nc):
*
* Pixel node registration used [Cartesian grid]
* Grid file format: nf = GMT netCDF format (float) (COARDS-compliant)
* x_min: -500000 x_max: 6500000 x_inc: 500 name: x nx: 14000
* y_min: -2500000 y_max: 1500000 y_inc: 500 name: y ny: 8000
* z_min: -6.48760175705 z_max: 16.1780223846 name: z
* scale_factor: 1 add_offset: 0
* mean: 0.00217145586762 stdev: 0.245563641735 rms: 0.245573241263
*/
//simulation area
float simulationArea[4];
simulationArea[0] = -450000;
simulationArea[1] = 6450000;
simulationArea[2] = -2450000;
simulationArea[3] = 1450000;
SWE_AsagiScenario l_scenario( ASAGI_INPUT_DIR "tohoku_gebco_ucsb3_500m_hawaii_bath.nc",
ASAGI_INPUT_DIR "tohoku_gebco_ucsb3_500m_hawaii_displ.nc",
(float) 28800., simulationArea);
#else
// create a simple artificial scenario
SWE_RadialDamBreakScenario l_scenario;
#endif
//! number of checkpoints for visualization (at each checkpoint in time, an output file is written).
int l_numberOfCheckPoints = 200;
//! size of a single cell in x- and y-direction
float l_dX, l_dY;
// compute the size of a single cell
l_dX = (l_scenario.getBoundaryPos(BND_RIGHT) - l_scenario.getBoundaryPos(BND_LEFT) )/l_nX;
l_dY = (l_scenario.getBoundaryPos(BND_TOP) - l_scenario.getBoundaryPos(BND_BOTTOM) )/l_nY;
// create a single wave propagation block
#ifndef CUDA
SWE_WavePropagationBlock l_wavePropgationBlock(l_nX,l_nY,l_dX,l_dY);
#else
SWE_WavePropagationBlockCuda l_wavePropgationBlock(l_nX,l_nY,l_dX,l_dY);
#endif
//! origin of the simulation domain in x- and y-direction
float l_originX, l_originY;
// get the origin from the scenario
l_originX = l_scenario.getBoundaryPos(BND_LEFT);
l_originY = l_scenario.getBoundaryPos(BND_BOTTOM);
// initialize the wave propagation block
l_wavePropgationBlock.initScenario(l_originX, l_originY, l_scenario);
//! time when the simulation ends.
float l_endSimulation = l_scenario.endSimulation();
//! checkpoints when output files are written.
float* l_checkPoints = new float[l_numberOfCheckPoints+1];
// compute the checkpoints in time
for(int cp = 0; cp <= l_numberOfCheckPoints; cp++) {
l_checkPoints[cp] = cp*(l_endSimulation/l_numberOfCheckPoints);
}
// Init fancy progressbar
tools::ProgressBar progressBar(l_endSimulation);
// write the output at time zero
tools::Logger::logger.printOutputTime((float) 0.);
progressBar.update(0.);
std::string l_fileName = generateBaseFileName(l_baseName,0,0);
//boundary size of the ghost layers
io::BoundarySize l_boundarySize = {{1, 1, 1, 1}};
#ifdef WRITENETCDF
//construct a NetCdfWriter
io::NetCdfWriter l_writer( l_fileName,
l_wavePropgationBlock.getBathymetry(),
l_boundarySize,
l_nX, l_nY,
l_dX, l_dY,
l_originX, l_originY);
#else
// consturct a VtkWriter
io::VtkWriter l_writer( l_fileName,
l_wavePropgationBlock.getBathymetry(),
l_boundarySize,
l_nX, l_nY,
l_dX, l_dY );
#endif
// Write zero time step
l_writer.writeTimeStep( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
(float) 0.);
/**
* Simulation.
*/
// print the start message and reset the wall clock time
progressBar.clear();
tools::Logger::logger.printStartMessage();
tools::Logger::logger.initWallClockTime(time(NULL));
//! simulation time.
float l_t = 0.0;
progressBar.update(l_t);
unsigned int l_iterations = 0;
// loop over checkpoints
for(int c=1; c<=l_numberOfCheckPoints; c++) {
// do time steps until next checkpoint is reached
while( l_t < l_checkPoints[c] ) {
// set values in ghost cells:
l_wavePropgationBlock.setGhostLayer();
// reset the cpu clock
tools::Logger::logger.resetClockToCurrentTime("Cpu");
// approximate the maximum time step
// TODO: This calculation should be replaced by the usage of the wave speeds occuring during the flux computation
// Remark: The code is executed on the CPU, therefore a "valid result" depends on the CPU-GPU-synchronization.
// l_wavePropgationBlock.computeMaxTimestep();
// compute numerical flux on each edge
l_wavePropgationBlock.computeNumericalFluxes();
//! maximum allowed time step width.
float l_maxTimeStepWidth = l_wavePropgationBlock.getMaxTimestep();
// update the cell values
l_wavePropgationBlock.updateUnknowns(l_maxTimeStepWidth);
// update the cpu time in the logger
tools::Logger::logger.updateTime("Cpu");
// update simulation time with time step width.
l_t += l_maxTimeStepWidth;
l_iterations++;
// print the current simulation time
progressBar.clear();
tools::Logger::logger.printSimulationTime(l_t);
progressBar.update(l_t);
}
// print current simulation time of the output
progressBar.clear();
tools::Logger::logger.printOutputTime(l_t);
progressBar.update(l_t);
// write output
l_writer.writeTimeStep( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
l_t);
}
/**
* Finalize.
*/
// write the statistics message
progressBar.clear();
tools::Logger::logger.printStatisticsMessage();
// print the cpu time
tools::Logger::logger.printTime("Cpu", "CPU time");
// print the wall clock time (includes plotting)
tools::Logger::logger.printWallClockTime(time(NULL));
// printer iteration counter
tools::Logger::logger.printIterationsDone(l_iterations);
return 0;
}
/**
* Main program for the simulation of dimensionalsplitting.
*/
int main( int argc, char** argv ) {
/**
* Initialization.
*/
// Parse command line parameters
tools::Args args;
#ifndef READXML
args.addOption("grid-size-x", 'x', "Number of cells in x direction");
args.addOption("grid-size-y", 'y', "Number of cells in y direction");
args.addOption("output-basepath", 'o', "Output base file name");
//add Options for Simulation Time and Boundary Condition arguments
args.addOption("simulated-time", 't', "Simulation time");
args.addOption("boundary-condition", 'b', "Boundary Condition");
#endif
tools::Args::Result ret = args.parse(argc, argv);
switch (ret)
{
case tools::Args::Error:
return 1;
case tools::Args::Help:
return 0;
}
//! number of grid cells in x- and y-direction.
int l_nX, l_nY;
//! l_baseName of the plots.
std::string l_baseName;
int l_simTime;
BoundaryType l_boundaryCond;
// read command line parameters
#ifndef READXML
l_nX = args.getArgument<int>("grid-size-x");
l_nY = args.getArgument<int>("grid-size-y");
l_baseName = args.getArgument<std::string>("output-basepath");
//read simulation-time and boundary-condition arguments
l_simTime= args.getArgument<int>("simulation-time");
std::string boundaryCondString= args.getArgument<std::string>("boundary-condition");
if(boundaryCondString.compare("WALL")){
l_boundaryCond=WALL;
}else if(boundaryCondString.compare("INFLOW")){
l_boundaryCond=INFLOW;
}else if(boundaryCondString.compare("CONNECT")){
l_boundaryCond=CONNECT;
}else if(boundaryCondString.compare("PASSIVE")){
l_boundaryCond=PASSIVE;
}else{//Outflow per default
l_boundaryCond=OUTFLOW;
}
#endif
// create a simple artificial scenario
//SWE_RadialDamBreakScenario l_scenario;
//SWE_ArtificialTsunamiScenario l_scenario;
//TsunamiScenario with NetCDF
TsunamiScenario l_scenario;
l_scenario.simTime=l_simTime;
l_scenario.boundaryCond=l_boundaryCond;
//! number of checkpoints for visualization (at each checkpoint in time, an output file is written).
int l_numberOfCheckPoints = 100;
//! size of a single cell in x- and y-direction
float l_dX, l_dY;
// compute the size of a single cell
l_dX = (l_scenario.getBoundaryPos(BND_RIGHT) - l_scenario.getBoundaryPos(BND_LEFT) )/l_nX;
l_dY = (l_scenario.getBoundaryPos(BND_TOP) - l_scenario.getBoundaryPos(BND_BOTTOM) )/l_nY;
// create a single wave propagation block
SWE_DimensionalSplitting l_wavePropgationBlock(l_nX,l_nY,l_dX,l_dY);
//! origin of the simulation domain in x- and y-direction
float l_originX, l_originY;
// get the origin from the scenario
l_originX = l_scenario.getBoundaryPos(BND_LEFT);
l_originY = l_scenario.getBoundaryPos(BND_BOTTOM);
// initialize the wave propagation block
l_wavePropgationBlock.initScenario(l_originX, l_originY, l_scenario);
//! time when the simulation ends.
float l_endSimulation = l_scenario.endSimulation();
//! checkpoints when output files are written.
float* l_checkPoints = new float[l_numberOfCheckPoints+1];
// compute the checkpoints in time
for(int cp = 0; cp <= l_numberOfCheckPoints; cp++) {
l_checkPoints[cp] = cp*(l_endSimulation/l_numberOfCheckPoints);
}
// Init fancy progressbar
//tools::ProgressBar progressBar(l_endSimulation);
// write the output at time zero
tools::Logger::logger.printOutputTime((float) 0.);
//progressBar.update(0.);
std::string l_fileName = generateBaseFileName(l_baseName,0,0);
//boundary size of the ghost layers
io::BoundarySize l_boundarySize = {{1, 1, 1, 1}};
// consturct a VtkWriter
io::VtkWriter l_writer( l_fileName,
l_wavePropgationBlock.getBathymetry(),
l_boundarySize,
l_nX, l_nY,
l_dX, l_dY );
// Write zero time step
l_writer.writeTimeStep( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
(float) 0.);
/**
* Simulation.
*/
// print the start message and reset the wall clock time
//progressBar.clear();
tools::Logger::logger.printStartMessage();
tools::Logger::logger.initWallClockTime(time(NULL));
//! simulation time.
float l_t = 0.0;
//progressBar.update(l_t);
unsigned int l_iterations = 0;
// loop over checkpoints
for(int c=1; c<=l_numberOfCheckPoints; c++) {
// do time steps until next checkpoint is reached
while( l_t < l_checkPoints[c] ) {
// set values in ghost cells:
l_wavePropgationBlock.setGhostLayer();
// reset the cpu clock
tools::Logger::logger.resetClockToCurrentTime("Cpu");
// compute numerical flux on each edge
l_wavePropgationBlock.computeNumericalFluxes();
//! maximum allowed time step width.
float l_maxTimeStepWidth = l_wavePropgationBlock.getMaxTimestep();
// update the cell values
l_wavePropgationBlock.updateUnknowns(l_maxTimeStepWidth);
// update the cpu time in the logger
tools::Logger::logger.updateTime("Cpu");
// update simulation time with time step width.
l_t += l_maxTimeStepWidth;
l_iterations++;
// print the current simulation time
//progressBar.clear();
tools::Logger::logger.printSimulationTime(l_t);
//progressBar.update(l_t);
}
// print current simulation time of the output
//progressBar.clear();
tools::Logger::logger.printOutputTime(l_t);
//progressBar.update(l_t);
// write output
l_writer.writeTimeStep( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
l_t);
}
/**
* Finalize.
*/
// write the statistics message
//progressBar.clear();
tools::Logger::logger.printStatisticsMessage();
// print the cpu time
tools::Logger::logger.printTime("Cpu", "CPU time");
// print the wall clock time (includes plotting)
tools::Logger::logger.printWallClockTime(time(NULL));
// printer iteration counter
tools::Logger::logger.printIterationsDone(l_iterations);
return 0;
}
/**
* Main program for the simulation on a single SWE_WavePropagationBlock.
*/
int main( int argc, char** argv ) {
/**
* Initialization.
*/
//! MPI Rank of a process.
int l_mpiRank;
//! number of MPI processes.
int l_numberOfProcesses;
// initialize MPI
if ( MPI_Init(&argc,&argv) != MPI_SUCCESS ) {
std::cerr << "MPI_Init failed." << std::endl;
}
// determine local MPI rank
MPI_Comm_rank(MPI_COMM_WORLD,&l_mpiRank);
// determine total number of processes
MPI_Comm_size(MPI_COMM_WORLD,&l_numberOfProcesses);
// initialize a logger for every MPI process
tools::Logger::logger.setProcessRank(l_mpiRank);
// print the welcome message
tools::Logger::logger.printWelcomeMessage();
// set current wall clock time within the solver
tools::Logger::logger.initWallClockTime( MPI_Wtime() );
//print the number of processes
tools::Logger::logger.printNumberOfProcesses(l_numberOfProcesses);
// check if the necessary command line input parameters are given
#ifndef READXML
std::vector<std::string> vargs;
vargs.push_back(argv[0]);
vargs.push_back("grid_size_x");
vargs.push_back("grid_size_y");
vargs.push_back("output_basepath");
vargs.push_back("output_steps_count");
#ifdef ASAGI
vargs.push_back("bathymetry_file");
vargs.push_back("displacement_file");
vargs.push_back("simul_area_min_x");
vargs.push_back("simul_area_max_x");
vargs.push_back("simul_area_min_y");
vargs.push_back("simul_area_max_y");
vargs.push_back("simul_duration_secs");
#endif
if (argc != vargs.size()) {
std::cout << "Usage: " << vargs[0];
for (int i = 1, e = vargs.size(); i != e; i++)
std::cout << " <" << vargs[i] << ">";
std::cout << std::endl << std::flush;
MPI_Finalize();
return 1;
}
#endif
//! total number of grid cell in x- and y-direction.
int l_nX, l_nY;
//! l_baseName of the plots.
std::string l_baseName;
// read command line parameters
#ifndef READXML
l_nX = atoi(ARG("grid_size_x"));
l_nY = atoi(ARG("grid_size_y"));
l_baseName = std::string(ARG("output_basepath"));
#endif
// read xml file
#ifdef READXML
assert(false); //TODO: not implemented.
if(argc != 2) {
l_sweLogger.printString("Aborting. Please provide a proper input file.");
l_sweLogger.printString("Example: ./SWE_gnu_debug_none_augrie config.xml");
return 1;
}
l_sweLogger.printString("Reading xml-file.");
std::string l_xmlFile = std::string(argv[1]);
l_sweLogger.printString(l_xmlFile);
CXMLConfig l_xmlConfig;
l_xmlConfig.loadConfig(l_xmlFile.c_str());
#endif // READXML
//! number of SWE_Blocks in x- and y-direction.
int l_blocksX, l_blocksY;
// determine the layout of MPI-ranks: use l_blocksX*l_blocksY grid blocks
l_blocksY = computeNumberOfBlockRows(l_numberOfProcesses);
l_blocksX = l_numberOfProcesses/l_blocksY;
// print information about the grid
tools::Logger::logger.printNumberOfCells(l_nX, l_nY);
tools::Logger::logger.printNumberOfBlocks(l_blocksX, l_blocksY);
//! local position of each MPI process in x- and y-direction.
int l_blockPositionX, l_blockPositionY;
// determine local block coordinates of each SWE_Block
l_blockPositionX = l_mpiRank / l_blocksY;
l_blockPositionY = l_mpiRank % l_blocksY;
#ifdef ASAGI
/*
* Pixel node registration used [Cartesian grid]
* Grid file format: nf = GMT netCDF format (float) (COARDS-compliant)
* x_min: -500000 x_max: 6500000 x_inc: 500 name: x nx: 14000
* y_min: -2500000 y_max: 1500000 y_inc: 500 name: y ny: 8000
* z_min: -6.48760175705 z_max: 16.1780223846 name: z
* scale_factor: 1 add_offset: 0
* mean: 0.00217145586762 stdev: 0.245563641735 rms: 0.245573241263
*/
//simulation area
float simulationArea[4];
simulationArea[0] = atof(ARG("simul_area_min_x"));
simulationArea[1] = atof(ARG("simul_area_max_x"));
simulationArea[2] = atof(ARG("simul_area_min_y"));
simulationArea[3] = atof(ARG("simul_area_max_y"));
float simulationDuration = atof(ARG("simul_duration_secs"));
SWE_AsagiScenario l_scenario(ARG("bathymetry_file"), ARG("displacement_file"),
simulationDuration, simulationArea);
#else
// create a simple artificial scenario
SWE_BathymetryDamBreakScenario l_scenario;
#endif
//! number of checkpoints for visualization (at each checkpoint in time, an output file is written).
int l_numberOfCheckPoints = atoi(ARG("output_steps_count"));
//! number of grid cells in x- and y-direction per process.
int l_nXLocal, l_nYLocal;
//! size of a single cell in x- and y-direction
float l_dX, l_dY;
// compute local number of cells for each SWE_Block
l_nXLocal = (l_blockPositionX < l_blocksX-1) ? l_nX/l_blocksX : l_nX - (l_blocksX-1)*(l_nX/l_blocksX);
l_nYLocal = (l_blockPositionY < l_blocksY-1) ? l_nY/l_blocksY : l_nY - (l_blocksY-1)*(l_nY/l_blocksY);
// compute the size of a single cell
l_dX = (l_scenario.getBoundaryPos(BND_RIGHT) - l_scenario.getBoundaryPos(BND_LEFT) )/l_nX;
l_dY = (l_scenario.getBoundaryPos(BND_TOP) - l_scenario.getBoundaryPos(BND_BOTTOM) )/l_nY;
// print information about the cell size and local number of cells
tools::Logger::logger.printCellSize(l_dX, l_dY);
tools::Logger::logger.printNumberOfCellsPerProcess(l_nXLocal, l_nYLocal);
//! origin of the simulation domain in x- and y-direction
float l_originX, l_originY;
// get the origin from the scenario
l_originX = l_scenario.getBoundaryPos(BND_LEFT) + l_blockPositionX*l_nXLocal*l_dX;;
l_originY = l_scenario.getBoundaryPos(BND_BOTTOM) + l_blockPositionY*l_nYLocal*l_dY;
// create a single wave propagation block
#ifndef CUDA
SWE_WavePropagationBlock l_wavePropgationBlock(l_nXLocal,l_nYLocal,l_dX,l_dY);
#else
//! number of CUDA devices per node TODO: hardcoded
int l_cudaDevicesPerNode = 7;
//! the id of the node local GPU
int l_cudaDeviceId = l_mpiRank % l_cudaDevicesPerNode;
SWE_BlockCUDA::init(l_cudaDeviceId);
SWE_WavePropagationBlockCuda l_wavePropgationBlock(l_nXLocal,l_nYLocal,l_dX,l_dY);
#endif
// initialize the wave propgation block
l_wavePropgationBlock.initScenario(l_originX, l_originY, l_scenario, true);
//! time when the simulation ends.
float l_endSimulation = l_scenario.endSimulation();
//! checkpoints when output files are written.
float* l_checkPoints = new float[l_numberOfCheckPoints+1];
// compute the checkpoints in time
for(int cp = 0; cp <= l_numberOfCheckPoints; cp++) {
l_checkPoints[cp] = cp*(l_endSimulation/l_numberOfCheckPoints);
}
/*
* Connect SWE blocks at boundaries
*/
// left and right boundaries
tools::Logger::logger.printString("Connecting SWE blocks at left boundaries.");
SWE_Block1D* l_leftInflow = l_wavePropgationBlock.grabGhostLayer(BND_LEFT);
SWE_Block1D* l_leftOutflow = l_wavePropgationBlock.registerCopyLayer(BND_LEFT);
if (l_blockPositionX == 0)
l_wavePropgationBlock.setBoundaryType(BND_LEFT, OUTFLOW);
tools::Logger::logger.printString("Connecting SWE blocks at right boundaries.");
SWE_Block1D* l_rightInflow = l_wavePropgationBlock.grabGhostLayer(BND_RIGHT);
SWE_Block1D* l_rightOutflow = l_wavePropgationBlock.registerCopyLayer(BND_RIGHT);
if (l_blockPositionX == l_blocksX-1)
l_wavePropgationBlock.setBoundaryType(BND_RIGHT, OUTFLOW);
// bottom and top boundaries
tools::Logger::logger.printString("Connecting SWE blocks at bottom boundaries.");
SWE_Block1D* l_bottomInflow = l_wavePropgationBlock.grabGhostLayer(BND_BOTTOM);
SWE_Block1D* l_bottomOutflow = l_wavePropgationBlock.registerCopyLayer(BND_BOTTOM);
if (l_blockPositionY == 0)
l_wavePropgationBlock.setBoundaryType(BND_BOTTOM, OUTFLOW);
tools::Logger::logger.printString("Connecting SWE blocks at top boundaries.");
SWE_Block1D* l_topInflow = l_wavePropgationBlock.grabGhostLayer(BND_TOP);
SWE_Block1D* l_topOutflow = l_wavePropgationBlock.registerCopyLayer(BND_TOP);
if (l_blockPositionY == l_blocksY-1)
l_wavePropgationBlock.setBoundaryType(BND_TOP, OUTFLOW);
/*
* The grid is stored column wise in memory:
*
* ************************** . . . **********
* * * ny+2 *2(ny+2)* * (ny+1)*
* * ny+1 * +ny+1 * +ny+1 * * (ny+2)*
* * * * * * +ny+1 *
* ************************** . . . **********
* * * * * * *
* . . . . . .
* . . . . . .
* . . . . . .
* * * * * * *
* ************************** . . . **********
* * * ny+2 *2(ny+2)* * (ny+1)*
* * 1 * +1 * +1 * * (ny+2)*
* * * * * * +1 *
* ************************** . . . **********
* * * ny+2 *2(ny+2)* * (ny+1)*
* * 0 * +0 * +0 * * (ny+2)*
* * * * * * +0 *
* ************************** . . . ***********
*
*
* -> The stride for a row is ny+2, because we have to jump over a whole column
* for every row-element. This holds only in the CPU-version, in CUDA a buffer is implemented.
* See SWE_BlockCUDA.hh/.cu for details.
* -> The stride for a column is 1, because we can access the elements linear in memory.
*/
//! MPI row-vector: l_nXLocal+2 blocks, 1 element per block, stride of l_nYLocal+2
MPI_Datatype l_mpiRow;
#ifndef CUDA
MPI_Type_vector(l_nXLocal+2, 1 , l_nYLocal+2, MPI_FLOAT, &l_mpiRow);
#else
MPI_Type_vector(1, l_nXLocal+2, 1 , MPI_FLOAT, &l_mpiRow);
#endif
MPI_Type_commit(&l_mpiRow);
//! MPI row-vector: 1 block, l_nYLocal+2 elements per block, stride of 1
MPI_Datatype l_mpiCol;
MPI_Type_vector(1, l_nYLocal+2, 1, MPI_FLOAT, &l_mpiCol);
MPI_Type_commit(&l_mpiCol);
//! MPI ranks of the neighbors
int l_leftNeighborRank, l_rightNeighborRank, l_bottomNeighborRank, l_topNeighborRank;
// compute MPI ranks of the neighbour processes
l_leftNeighborRank = (l_blockPositionX > 0) ? l_mpiRank-l_blocksY : MPI_PROC_NULL;
l_rightNeighborRank = (l_blockPositionX < l_blocksX-1) ? l_mpiRank+l_blocksY : MPI_PROC_NULL;
l_bottomNeighborRank = (l_blockPositionY > 0) ? l_mpiRank-1 : MPI_PROC_NULL;
l_topNeighborRank = (l_blockPositionY < l_blocksY-1) ? l_mpiRank+1 : MPI_PROC_NULL;
// print the MPI grid
tools::Logger::logger.cout() << "neighbors: "
<< l_leftNeighborRank << " (left), "
<< l_rightNeighborRank << " (right), "
<< l_bottomNeighborRank << " (bottom), "
<< l_topNeighborRank << " (top)" << std::endl;
// intially exchange ghost and copy layers
exchangeLeftRightGhostLayers( l_leftNeighborRank, l_leftInflow, l_leftOutflow,
l_rightNeighborRank, l_rightInflow, l_rightOutflow,
l_mpiCol );
exchangeBottomTopGhostLayers( l_bottomNeighborRank, l_bottomInflow, l_bottomOutflow,
l_topNeighborRank, l_topInflow, l_topOutflow,
l_mpiRow );
// Init fancy progressbar
tools::ProgressBar progressBar(l_endSimulation, l_mpiRank);
// write the output at time zero
tools::Logger::logger.printOutputTime(0);
progressBar.update(0.);
std::string l_fileName = generateBaseFileName(l_baseName,l_blockPositionX,l_blockPositionY);
//boundary size of the ghost layers
io::BoundarySize l_boundarySize = {{1, 1, 1, 1}};
#ifdef WRITENETCDF
//construct a NetCdfWriter
io::NetCdfWriter l_writer( l_fileName,
l_wavePropgationBlock.getBathymetry(),
l_boundarySize,
l_nXLocal, l_nYLocal,
l_dX, l_dY,
l_originX, l_originY );
#else
// Construct a VtkWriter
io::VtkWriter l_writer( l_fileName,
l_wavePropgationBlock.getBathymetry(),
l_boundarySize,
l_nXLocal, l_nYLocal,
l_dX, l_dY,
l_blockPositionX*l_nXLocal, l_blockPositionY*l_nYLocal );
#endif
// Write zero time step
l_writer.writeTimeStep( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
(float) 0.);
/**
* Simulation.
*/
// print the start message and reset the wall clock time
progressBar.clear();
tools::Logger::logger.printStartMessage();
tools::Logger::logger.initWallClockTime(time(NULL));
//! simulation time.
float l_t = 0.0;
progressBar.update(l_t);
unsigned int l_iterations = 0;
// loop over checkpoints
for(int c=1; c<=l_numberOfCheckPoints; c++) {
// do time steps until next checkpoint is reached
while( l_t < l_checkPoints[c] ) {
//reset CPU-Communication clock
tools::Logger::logger.resetCpuCommunicationClockToCurrentTime();
// exchange ghost and copy layers
exchangeLeftRightGhostLayers( l_leftNeighborRank, l_leftInflow, l_leftOutflow,
l_rightNeighborRank, l_rightInflow, l_rightOutflow,
l_mpiCol );
exchangeBottomTopGhostLayers( l_bottomNeighborRank, l_bottomInflow, l_bottomOutflow,
l_topNeighborRank, l_topInflow, l_topOutflow,
l_mpiRow );
// reset the cpu clock
tools::Logger::logger.resetCpuClockToCurrentTime();
// set values in ghost cells
l_wavePropgationBlock.setGhostLayer();
// compute numerical flux on each edge
l_wavePropgationBlock.computeNumericalFluxes();
//! maximum allowed time step width within a block.
float l_maxTimeStepWidth = l_wavePropgationBlock.getMaxTimestep();
// update the cpu time in the logger
tools::Logger::logger.updateCpuTime();
//! maximum allowed time steps of all blocks
float l_maxTimeStepWidthGlobal;
// determine smallest time step of all blocks
MPI_Allreduce(&l_maxTimeStepWidth, &l_maxTimeStepWidthGlobal, 1, MPI_FLOAT, MPI_MIN, MPI_COMM_WORLD);
// reset the cpu time
tools::Logger::logger.resetCpuClockToCurrentTime();
// update the cell values
l_wavePropgationBlock.updateUnknowns(l_maxTimeStepWidthGlobal);
// update the cpu and CPU-communication time in the logger
tools::Logger::logger.updateCpuTime();
tools::Logger::logger.updateCpuCommunicationTime();
// update simulation time with time step width.
l_t += l_maxTimeStepWidthGlobal;
l_iterations++;
// print the current simulation time
progressBar.clear();
tools::Logger::logger.printSimulationTime(l_t);
progressBar.update(l_t);
}
// print current simulation time
progressBar.clear();
tools::Logger::logger.printOutputTime(l_t);
progressBar.update(l_t);
// write output
l_writer.writeTimeStep( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
l_t);
}
/**
* Finalize.
*/
#ifdef ASAGI
// Free ASAGI resources
l_scenario.deleteGrids();
#endif
progressBar.clear();
// write the statistics message
tools::Logger::logger.printStatisticsMessage();
// print the cpu time
tools::Logger::logger.printCpuTime("CPU time");
// print CPU + Communication time
tools::Logger::logger.printCpuCommunicationTime();
// print the wall clock time (includes plotting)
tools::Logger::logger.printWallClockTime(time(NULL));
// printer iteration counter
tools::Logger::logger.printIterationsDone(l_iterations);
// print the finish message
tools::Logger::logger.printFinishMessage();
// finalize MPI execution
MPI_Finalize();
return 0;
}
/**
* Main program for the simulation on a single SWE_WavePropagationBlock.
*/
int main( int argc, char** argv ) {
/**
* Initialization.
*/
// check if the necessary command line input parameters are given
#ifndef READXML
if(argc != 4) {
std::cout << "Aborting ... please provide proper input parameters." << std::endl
<< "Example: ./SWE_parallel 200 300 /work/openmp_out" << std::endl
<< "\tfor a single block of size 200 * 300" << std::endl;
return 1;
}
#endif
//! number of grid cells in x- and y-direction.
int l_nX, l_nY;
//! l_baseName of the plots.
std::string l_baseName;
// read command line parameters
#ifndef READXML
l_nY = l_nX = atoi(argv[1]);
l_nY = atoi(argv[2]);
l_baseName = std::string(argv[3]);
#endif
// read xml file
#ifdef READXML
assert(false); //TODO: not implemented.
if(argc != 2) {
s_sweLogger.printString("Aborting. Please provide a proper input file.");
s_sweLogger.printString("Example: ./SWE_gnu_debug_none_augrie config.xml");
return 1;
}
s_sweLogger.printString("Reading xml-file.");
std::string l_xmlFile = std::string(argv[1]);
s_sweLogger.printString(l_xmlFile);
CXMLConfig l_xmlConfig;
l_xmlConfig.loadConfig(l_xmlFile.c_str());
#endif
#ifdef ASAGI
/* Information about the example bathymetry grid (tohoku_gebco_ucsb3_500m_hawaii_bath.nc):
*
* Pixel node registration used [Cartesian grid]
* Grid file format: nf = GMT netCDF format (float) (COARDS-compliant)
* x_min: -500000 x_max: 6500000 x_inc: 500 name: x nx: 14000
* y_min: -2500000 y_max: 1500000 y_inc: 500 name: y ny: 8000
* z_min: -6.48760175705 z_max: 16.1780223846 name: z
* scale_factor: 1 add_offset: 0
* mean: 0.00217145586762 stdev: 0.245563641735 rms: 0.245573241263
*/
//simulation area
float simulationArea[4];
simulationArea[0] = -450000;
simulationArea[1] = 6450000;
simulationArea[2] = -2450000;
simulationArea[3] = 1450000;
SWE_AsagiScenario l_scenario( "/work/breuera/workspace/geo_information/output/tohoku_gebco_ucsb3_500m_hawaii_bath.nc",
"/work/breuera/workspace/geo_information/output/tohoku_gebco_ucsb3_500m_hawaii_displ.nc",
(float) 28800., simulationArea);
#else
// create a simple artificial scenario
SWE_BathymetryDamBreakScenario l_scenario;
#endif
//! number of checkpoints for visualization (at each checkpoint in time, an output file is written).
int l_numberOfCheckPoints = 20;
//! size of a single cell in x- and y-direction
float l_dX, l_dY;
// compute the size of a single cell
l_dX = (l_scenario.getBoundaryPos(BND_RIGHT) - l_scenario.getBoundaryPos(BND_LEFT) )/l_nX;
l_dY = (l_scenario.getBoundaryPos(BND_TOP) - l_scenario.getBoundaryPos(BND_BOTTOM) )/l_nY;
// initialize the grid data and the corresponding static variables
SWE_Block::initGridData(l_nX,l_nY,l_dX,l_dY);
//! origin of the simulation domain in x- and y-direction
float l_originX, l_originY;
// get the origin from the scenario
l_originX = l_scenario.getBoundaryPos(BND_LEFT);
l_originY = l_scenario.getBoundaryPos(BND_BOTTOM);
// create a single wave propagation block
#ifndef CUDA
SWE_WavePropagationBlock l_wavePropgationBlock(l_originX, l_originY);
#else
SWE_WavePropagationBlockCuda l_wavePropgationBlock(l_originX, l_originY);
#endif
// initialize the wave propgation block
l_wavePropgationBlock.initScenario(l_scenario);
//! time when the simulation ends.
float l_endSimulation = l_scenario.endSimulation();
//! checkpoints when output files are written.
float* l_checkPoints = new float[l_numberOfCheckPoints+1];
// compute the checkpoints in time
for(int cp = 0; cp <= l_numberOfCheckPoints; cp++) {
l_checkPoints[cp] = cp*(l_endSimulation/l_numberOfCheckPoints);
}
// write the output at time zero
s_sweLogger.printOutputTime((float) 0.);
#ifdef WRITENETCDF
//boundary size of the ghost layers
int l_boundarySize[4];
l_boundarySize[0] = l_boundarySize[1] = l_boundarySize[2] = l_boundarySize[3] = 1;
//construct a NetCdfWriter
std::string l_fileName = l_baseName;
io::NetCdfWriter l_netCdfWriter( l_fileName, l_nX, l_nY );
//create the netCDF-file
l_netCdfWriter.createNetCdfFile(l_dX, l_dY, l_originX, l_originY);
l_netCdfWriter.writeBathymetry(l_wavePropgationBlock.getBathymetry(), l_boundarySize);
l_netCdfWriter.writeUnknowns( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
l_boundarySize, (float) 0.);
#else
l_wavePropgationBlock.writeVTKFileXML(generateFileName(l_baseName,0,0,0), l_nX, l_nY);
#endif
/**
* Simulation.
*/
// print the start message and reset the wall clock time
s_sweLogger.printStartMessage();
s_sweLogger.initWallClockTime(time(NULL));
//! simulation time.
float l_t = 0.0;
// loop over checkpoints
for(int c=1; c<=l_numberOfCheckPoints; c++) {
// reset the cpu clock
s_sweLogger.resetCpuClockToCurrentTime();
// do time steps until next checkpoint is reached
while( l_t < l_checkPoints[c] ) {
// set values in ghost cells:
l_wavePropgationBlock.setGhostLayer();
// compute numerical flux on each edge
l_wavePropgationBlock.computeNumericalFluxes();
//! maximum allowed time step width.
float l_maxTimeStepWidth = l_wavePropgationBlock.getMaxTimestep();
// update the cell values
l_wavePropgationBlock.updateUnknowns(l_maxTimeStepWidth);
// update simulation time with time step width.
l_t += l_maxTimeStepWidth;
// print the current simulation time
s_sweLogger.printSimulationTime(l_t);
}
// update the cpu time in the logger
s_sweLogger.updateCpuTime();
// print current simulation time of the output
s_sweLogger.printOutputTime(l_t);
// write output
#ifdef WRITENETCDF
l_netCdfWriter.writeUnknowns( l_wavePropgationBlock.getWaterHeight(),
l_wavePropgationBlock.getDischarge_hu(),
l_wavePropgationBlock.getDischarge_hv(),
l_boundarySize, l_t);
#else
l_wavePropgationBlock.writeVTKFileXML(generateFileName(l_baseName,c,0,0), l_nX, l_nY);
#endif
}
/**
* Finalize.
*/
// write the statistics message
s_sweLogger.printStatisticsMessage();
// print the cpu time
s_sweLogger.printCpuTime();
// print the wall clock time (includes plotting)
s_sweLogger.printWallClockTime(time(NULL));
return 0;
}
