void seissol::checkpoint::mpio::WavefieldAsync::writePrepare(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_writePrepare");
SCOREP_USER_REGION("CheckPoint_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION);
// Write the header
writeHeader(time, timestepWaveField);
// Create copy of the dofs
memcpy(m_dofsCopy, dofs(), numDofs()*sizeof(real));
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_begin_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all_begin(file(), m_dofsCopy, numDofs(), MPI_DOUBLE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
m_started = true;
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
// Write the header
writeHeader(time, timestepWaveField);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all(file(), dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::checkpoint::sionlib::Wavefield::write(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
int file = open(dataFile(odd()), writeMode());
checkErr(file);
// Write the header
SCOREP_USER_REGION_DEFINE(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(sion_coll_fwrite(header, headerSize, 1, file), 1);
SCOREP_USER_REGION_END(r_write_header);
// Save data
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(sion_coll_fwrite(dofs(), sizeof(real), numDofs(), file), numDofs());
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint(file);
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::FaultAsync::writePrepare(int timestepFault)
{
EPIK_TRACER("CheckPointFault_writePrepare");
SCOREP_USER_REGION("CheckPointFault_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
// Write the header
writeHeader(timestepFault);
// Create copy of the data
for (unsigned int i = 0; i < NUM_VARIABLES; i++)
memcpy(&m_dataCopy[i*numSides()*numBndGP()],
data(i), numSides()*numBndGP()*sizeof(double));
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_fault");
SCOREP_USER_REGION_DEFINE(r_write_fault);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_fault, "checkpoint_write_begin_fault", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all_begin(file(), m_dataCopy, numSides() * numBndGP() * NUM_VARIABLES, MPI_DOUBLE));
EPIK_USER_END(r_write_fault);
SCOREP_USER_REGION_END(r_write_fault);
m_started = true;
logInfo(rank()) << "Writing fault check point. Done.";
}
void seissol::checkpoint::mpio::Wavefield::writeHeader(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("checkpoint_write_header", SCOREP_USER_REGION_TYPE_FUNCTION);
checkMPIErr(setHeaderView(file()));
if (rank() == 0)
checkMPIErr(MPI_File_write(file(), const_cast<void*>(header), 1, headerType(), MPI_STATUS_IGNORE));
}
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
// Start at the beginning
checkErr(lseek64(file(), 0, SEEK_SET));
// Write the header
EPIK_USER_REG(r_write_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_write_header);
EPIK_USER_START(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
WavefieldHeader header;
header.time = time;
header.timestepWaveField = timestepWaveField;
writeHeader(file(), header);
EPIK_USER_END(r_write_header);
SCOREP_USER_REGION_END(r_write_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Convert to char* to do pointer arithmetic
const char* buffer = reinterpret_cast<const char*>(dofs());
unsigned long left = numDofs()*sizeof(real);
if (alignment()) {
left = (left + alignment() - 1) / alignment();
left *= alignment();
}
while (left > 0) {
unsigned long written = ::write(file(), buffer, left);
if (written <= 0)
checkErr(written, left);
buffer += written;
left -= written;
}
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::WavefieldAsync::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
if (m_started)
checkMPIErr(MPI_File_write_all_end(file(), m_dofsCopy, MPI_STATUS_IGNORE));
// Finalize the checkpoint
finalizeCheckpoint();
}
int main(int argc, char* argv[])
{
EPIK_TRACER("SeisSol");
SCOREP_USER_REGION("SeisSol", SCOREP_USER_REGION_TYPE_FUNCTION);
// Initialize SeisSol
seissol::SeisSol::main.init(argc, argv);
// Initialize Fortan Part and run SeisSol
fortran_main();
// Finalize SeisSol
seissol::SeisSol::main.finalize();
}
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
// Skip identifier
checkErr(lseek64(file(), sizeof(unsigned long), SEEK_SET));
// Write the header
EPIK_USER_REG(r_write_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_write_header);
EPIK_USER_START(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(::write(file(), &time, sizeof(time)), sizeof(time));
checkErr(::write(file(), ×tepWaveField, sizeof(timestepWaveField)),
sizeof(timestepWaveField));
EPIK_USER_END(r_write_header);
SCOREP_USER_REGION_END(r_write_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Convert to char* to do pointer arithmetic
const char* buffer = reinterpret_cast<const char*>(dofs());
unsigned long left = numDofs()*sizeof(real);
while (left > 0) {
unsigned long written = ::write(file(), buffer, left);
if (written <= 0)
checkErr(written, left);
buffer += written;
left -= written;
}
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::checkpoint::mpio::FaultAsync::write(int timestepFault)
{
EPIK_TRACER("CheckPointFault_write");
SCOREP_USER_REGION("CheckPointFault_write", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
logInfo(rank()) << "Writing fault check point.";
if (m_started)
checkMPIErr(MPI_File_write_all_end(file(), m_dataCopy, MPI_STATUS_IGNORE));
// Finalize the checkpoint
finalizeCheckpoint();
}
void seissol::checkpoint::mpio::Wavefield::writeHeader(double time, int timestepWaveField)
{
EPIK_TRACER("checkpoint_write_header");
SCOREP_USER_REGION("checkpoint_write_header", SCOREP_USER_REGION_TYPE_FUNCTION);
checkMPIErr(setHeaderView(file()));
if (rank() == 0) {
Header header;
header.identifier = identifier();
header.partitions = partitions();
header.time = time;
header.timestepWavefield = timestepWaveField;
checkMPIErr(MPI_File_write(file(), &header, 1, headerType(), MPI_STATUS_IGNORE));
}
}
void seissol::checkpoint::mpio::Wavefield::write(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
// Write the header
writeHeader(header, headerSize);
// Save data
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
unsigned int totalIter = totalIterations();
unsigned int iter = iterations();
unsigned int count = dofsPerIteration();
if (m_useLargeBuffer) {
totalIter = (totalIter + sizeof(real) - 1) / sizeof(real);
iter = (iter + sizeof(real) - 1) / sizeof(real);
count *= sizeof(real);
}
unsigned long offset = 0;
for (unsigned int i = 0; i < totalIter; i++) {
if (i == iter-1)
// Last iteration
count = numDofs() - (iter-1) * count;
checkMPIErr(MPI_File_write_all(file(), const_cast<real*>(&dofs()[offset]), count, MPI_DOUBLE, MPI_STATUS_IGNORE));
if (i < iter-1)
offset += count;
// otherwise we just continue writing the last chunk over and over
else if (i != totalIter-1)
checkMPIErr(MPI_File_seek(file(), -count * sizeof(real), MPI_SEEK_CUR));
}
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::h5::Fault::write(int timestepFault)
{
EPIK_TRACER("CheckPointFault_write");
SCOREP_USER_REGION("CheckPointFault_write", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
logInfo(rank()) << "Writing fault check point.";
// Create array with all pointers
EPIK_USER_REG(r_write_fault, "checkpoint_write_fault");
SCOREP_USER_REGION_DEFINE(r_write_fault);
EPIK_USER_START(r_write_fault);
SCOREP_USER_REGION_BEGIN(r_write_fault, "checkpoint_write_fault", SCOREP_USER_REGION_TYPE_COMMON);
// Attributes
checkH5Err(H5Awrite(m_h5timestepFault[odd()], H5T_NATIVE_INT, ×tepFault));
// Set memory and file space
hsize_t fStart[2] = {fileOffset(), 0};
hsize_t count[2] = {numSides(), numBndGP()};
hid_t h5memSpace = H5Screate_simple(2, count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, fStart, 0L, count, 0L));
for (unsigned int i = 0; i < NUM_VARIABLES; i++) {
checkH5Err(H5Dwrite(m_h5data[odd()][i], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), data(i)));
}
checkH5Err(H5Sclose(h5memSpace));
EPIK_USER_END(r_write_fault);
SCOREP_USER_REGION_END(r_write_fault);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing fault check point. Done.";
}
void seissol::checkpoint::h5::Wavefield::write(double time, int waveFieldTimeStep)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
EPIK_USER_REG(r_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_header);
EPIK_USER_START(r_header);
SCOREP_USER_REGION_BEGIN(r_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
// Time
checkH5Err(H5Awrite(m_h5time[odd()], H5T_NATIVE_DOUBLE, &time));
// Wavefield writer
checkH5Err(H5Awrite(m_h5timestepWavefield[odd()], H5T_NATIVE_INT, &waveFieldTimeStep));
EPIK_USER_END(r_header);
SCOREP_USER_REGION_END(r_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Write the wave field
unsigned int offset = 0;
hsize_t fStart = fileOffset();
hsize_t count = dofsPerIteration();
hid_t h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
for (unsigned int i = 0; i < totalIterations()-1; i++) {
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &const_cast<real*>(dofs())[offset]));
// We are finished in less iterations, read data twice
// so everybody needs the same number of iterations
if (i < iterations()-1) {
fStart += count;
offset += count;
}
}
checkH5Err(H5Sclose(h5memSpace));
// Save reminding data in the last iteration
count = numDofs() - (iterations() - 1) * count;
h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &dofs()[offset]));
checkH5Err(H5Sclose(h5memSpace));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::time_stepping::TimeManager::addClusters( struct TimeStepping &i_timeStepping,
struct MeshStructure *i_meshStructure,
struct CellLocalInformation *io_cellLocalInformation,
unsigned int (*i_meshToClusters)[2] ) {
SCOREP_USER_REGION( "addClusters", SCOREP_USER_REGION_TYPE_FUNCTION )
#ifdef USE_MPI
// init mpi rank
MPI_Comm_rank( MPI_COMM_WORLD, &m_mpiRank);
#endif
// assert non-zero pointers
assert( i_meshStructure != NULL );
assert( io_cellLocalInformation != NULL );
assert( i_meshToClusters != NULL );
// store the time stepping
m_timeStepping = i_timeStepping;
// store mesh to clusters mapping
m_meshToClusters = i_meshToClusters;
// derive lts setups
seissol::initializers::time_stepping::deriveLtsSetups( m_timeStepping.numberOfLocalClusters,
i_meshStructure,
io_cellLocalInformation );
// initialize memory layout
m_memoryManager.initializeMemoryLayout( m_timeStepping, i_meshStructure, io_cellLocalInformation );
// iterate over local time clusters
for( unsigned int l_cluster = 0; l_cluster < m_timeStepping.numberOfLocalClusters; l_cluster++ ) {
struct MeshStructure *l_meshStructure = NULL;
#ifdef USE_MPI
struct CellLocalInformation *l_copyCellInformation = NULL;
#endif
struct CellLocalInformation *l_interiorCellInformation = NULL;
struct GlobalData *l_globalData = NULL;
#ifdef USE_MPI
struct CellData *l_copyCellData = NULL;
#endif
struct CellData *l_interiorCellData = NULL;
struct Cells *l_cells = NULL;
// get memory layout of this cluster
m_memoryManager.getMemoryLayout( l_cluster,
l_meshStructure,
#ifdef USE_MPI
l_copyCellInformation,
#endif
l_interiorCellInformation,
l_globalData,
#ifdef USE_MPI
l_copyCellData,
#endif
l_interiorCellData,
l_cells );
// add this time cluster
m_clusters.push_back( new TimeCluster( l_cluster,
m_timeStepping.clusterIds[l_cluster],
m_timeKernel,
m_volumeKernel,
m_boundaryKernel,
l_meshStructure,
#ifdef USE_MPI
l_copyCellInformation,
#endif
l_interiorCellInformation,
l_globalData,
#ifdef USE_MPI
l_copyCellData,
#endif
l_interiorCellData,
l_cells ) );
}
}
