void seissol::checkpoint::mpio::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
// Write the header
writeHeader(time, timestepWaveField);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all(file(), dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::checkpoint::mpio::WavefieldAsync::writePrepare(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_writePrepare");
SCOREP_USER_REGION("CheckPoint_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION);
// Write the header
writeHeader(time, timestepWaveField);
// Create copy of the dofs
memcpy(m_dofsCopy, dofs(), numDofs()*sizeof(real));
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_begin_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all_begin(file(), m_dofsCopy, numDofs(), MPI_DOUBLE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
m_started = true;
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::FaultAsync::writePrepare(int timestepFault)
{
EPIK_TRACER("CheckPointFault_writePrepare");
SCOREP_USER_REGION("CheckPointFault_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
// Write the header
writeHeader(timestepFault);
// Create copy of the data
for (unsigned int i = 0; i < NUM_VARIABLES; i++)
memcpy(&m_dataCopy[i*numSides()*numBndGP()],
data(i), numSides()*numBndGP()*sizeof(double));
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_fault");
SCOREP_USER_REGION_DEFINE(r_write_fault);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_fault, "checkpoint_write_begin_fault", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all_begin(file(), m_dataCopy, numSides() * numBndGP() * NUM_VARIABLES, MPI_DOUBLE));
EPIK_USER_END(r_write_fault);
SCOREP_USER_REGION_END(r_write_fault);
m_started = true;
logInfo(rank()) << "Writing fault check point. Done.";
}
void seissol::checkpoint::mpio::Wavefield::load(double &time, int ×tepWaveField)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::load();
MPI_File file = open();
if (file == MPI_FILE_NULL)
logError() << "Could not open checkpoint file";
// Read and broadcast header
checkMPIErr(setHeaderView(file));
Header header;
if (rank() == 0)
checkMPIErr(MPI_File_read(file, &header, 1, headerType(), MPI_STATUS_IGNORE));
MPI_Bcast(&header, 1, headerType(), 0, comm());
time = header.time;
timestepWaveField = header.timestepWavefield;
// Read dofs
checkMPIErr(setDataView(file));
checkMPIErr(MPI_File_read_all(file, dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
// Close the file
checkMPIErr(MPI_File_close(&file));
}
void seissol::checkpoint::mpio::Wavefield::load(real* dofs)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::setLoaded();
MPI_File file = open();
if (file == MPI_FILE_NULL)
logError() << "Could not open checkpoint file";
// Read and broadcast header
checkMPIErr(setHeaderView(file));
if (rank() == 0)
checkMPIErr(MPI_File_read(file, header().data(), 1, headerType(), MPI_STATUS_IGNORE));
MPI_Bcast(header().data(), 1, headerType(), 0, comm());
// Read dofs
checkMPIErr(setDataView(file));
checkMPIErr(MPI_File_read_all(file, dofs, numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
// Close the file
checkMPIErr(MPI_File_close(&file));
}
void seissol::checkpoint::mpio::Wavefield::writeHeader(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("checkpoint_write_header", SCOREP_USER_REGION_TYPE_FUNCTION);
checkMPIErr(setHeaderView(file()));
if (rank() == 0)
checkMPIErr(MPI_File_write(file(), const_cast<void*>(header), 1, headerType(), MPI_STATUS_IGNORE));
}
void seissol::checkpoint::mpio::Wavefield::writeHeader(double time, int timestepWaveField)
{
EPIK_TRACER("checkpoint_write_header");
SCOREP_USER_REGION("checkpoint_write_header", SCOREP_USER_REGION_TYPE_FUNCTION);
checkMPIErr(setHeaderView(file()));
if (rank() == 0) {
Header header;
header.identifier = identifier();
header.partitions = partitions();
header.time = time;
header.timestepWavefield = timestepWaveField;
checkMPIErr(MPI_File_write(file(), &header, 1, headerType(), MPI_STATUS_IGNORE));
}
}
void seissol::checkpoint::mpio::Wavefield::write(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
// Write the header
writeHeader(header, headerSize);
// Save data
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
unsigned int totalIter = totalIterations();
unsigned int iter = iterations();
unsigned int count = dofsPerIteration();
if (m_useLargeBuffer) {
totalIter = (totalIter + sizeof(real) - 1) / sizeof(real);
iter = (iter + sizeof(real) - 1) / sizeof(real);
count *= sizeof(real);
}
unsigned long offset = 0;
for (unsigned int i = 0; i < totalIter; i++) {
if (i == iter-1)
// Last iteration
count = numDofs() - (iter-1) * count;
checkMPIErr(MPI_File_write_all(file(), const_cast<real*>(&dofs()[offset]), count, MPI_DOUBLE, MPI_STATUS_IGNORE));
if (i < iter-1)
offset += count;
// otherwise we just continue writing the last chunk over and over
else if (i != totalIter-1)
checkMPIErr(MPI_File_seek(file(), -count * sizeof(real), MPI_SEEK_CUR));
}
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::WavefieldAsync::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
if (m_started)
checkMPIErr(MPI_File_write_all_end(file(), m_dofsCopy, MPI_STATUS_IGNORE));
// Finalize the checkpoint
finalizeCheckpoint();
}
void seissol::checkpoint::mpio::FaultAsync::write(int timestepFault)
{
EPIK_TRACER("CheckPointFault_write");
SCOREP_USER_REGION("CheckPointFault_write", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
logInfo(rank()) << "Writing fault check point.";
if (m_started)
checkMPIErr(MPI_File_write_all_end(file(), m_dataCopy, MPI_STATUS_IGNORE));
// Finalize the checkpoint
finalizeCheckpoint();
}
