bool seissol::checkpoint::h5::Wavefield::validate(hid_t h5file) const
{
// Turn of error printing
H5ErrHandler errHandler;
// Check #partitions
hid_t h5attr = H5Aopen(h5file, "partitions", H5P_DEFAULT);
if (h5attr < 0) {
logWarning(rank()) << "Checkpoint does not have a partition attribute.";
return false;
}
int p;
herr_t err = H5Aread(h5attr, H5T_NATIVE_INT, &p);
checkH5Err(H5Aclose(h5attr));
if (err < 0 || p != partitions()) {
logWarning(rank()) << "Partitions in checkpoint do not match.";
return false;
}
// Check dimensions
hid_t h5data = H5Dopen(h5file, "values", H5P_DEFAULT);
if (h5data < 0) {
logWarning(rank()) << "Checkpoint does not contains a data array.";
return false;
}
hid_t h5space = H5Dget_space(h5data);
checkH5Err(H5Dclose(h5data));
if (h5space < 0) {
logWarning(rank()) << "Could not get space identifier in checkpoint.";
return false;
}
bool isValid = true;
int dims = H5Sget_simple_extent_ndims(h5space);
if (dims != 1) {
isValid = false;
logWarning(rank()) << "Number of dimensions in checkpoint does not match.";
} else {
hsize_t dimSize;
if (H5Sget_simple_extent_dims(h5space, &dimSize, 0L) != 1) {
isValid = false;
logWarning(rank()) << "Could not get dimension sizes of checkpoint.";
} else {
if (dimSize != numTotalElems()) {
isValid = false;
logWarning(rank()) << "Number of elements in checkpoint does not match.";
}
}
}
checkH5Err(H5Sclose(h5space));
return isValid;
}
bool seissol::checkpoint::h5::Fault::validate(hid_t h5file) const
{
// Turn of error printing
H5ErrHandler errHandler;
// Check dimensions
for (unsigned int i = 0; i < NUM_VARIABLES; i++) {
hid_t h5data = H5Dopen(h5file, VAR_NAMES[i], H5P_DEFAULT);
if (h5data < 0) {
logWarning(rank()) << "Dataset" << VAR_NAMES[i] << "not found in checkpoint.";
return false;
}
hid_t h5space = H5Dget_space(h5data);
checkH5Err(H5Dclose(h5data));
if (h5space < 0) {
logWarning(rank()) << "Could not get space identifier for" << VAR_NAMES[i] << "in checkpoint.";
return false;
}
bool isValid = true;
int dims = H5Sget_simple_extent_ndims(h5space);
if (dims != 2) {
isValid = false;
logWarning() << "Number of dimensions for" << VAR_NAMES[i] << "in checkpoint does not match.";
} else {
hsize_t dimSize[2];
if (H5Sget_simple_extent_dims(h5space, dimSize, 0L) != 2) {
isValid = false;
logWarning(rank()) << "Could not get dimension sizes for" << VAR_NAMES[i] << "of checkpoint.";
} else {
if (dimSize[0] != numTotalElems()) {
isValid = false;
logWarning(rank()) << "Number of elements for" << VAR_NAMES[i] << "in checkpoint does not match.";
}
if (dimSize[1] != numBndGP()) {
logWarning(rank()) << "Number of boundary points for" << VAR_NAMES[i] << "in checkpoint does not match.";
isValid = false;
}
}
}
checkH5Err(H5Sclose(h5space));
// Dimensions for at least one variable do not match -> invalid
if (!isValid)
return false;
}
return true;
}
bool seissol::checkpoint::h5::Wavefield::init(unsigned int numDofs, unsigned int groupSize)
{
seissol::checkpoint::Wavefield::init(numDofs, groupSize);
// Data space for the file
hsize_t fileSize = numTotalElems();
m_h5fSpaceData = H5Screate_simple(1, &fileSize, 0L);
checkH5Err(m_h5fSpaceData);
setupXferList();
return exists();
}
bool seissol::checkpoint::h5::Fault::init(
double* mu, double* slipRate1, double* slipRate2, double* slip1, double* slip2,
double* state, double* strength,
unsigned int numSides, unsigned int numBndGP)
{
seissol::checkpoint::Fault::init(mu, slipRate1, slipRate2, slip1, slip2, state, strength,
numSides, numBndGP);
if (numSides == 0)
return true;
// Compute total number of cells and local offset
setSumOffset(numSides);
// Dataspace for the file
hsize_t fileSize[2] = {numTotalElems(), numBndGP};
m_h5fSpaceData = H5Screate_simple(2, fileSize, 0L);
checkH5Err(m_h5fSpaceData);
setupXferList();
return exists();
}
void seissol::checkpoint::h5::Fault::write(int timestepFault)
{
EPIK_TRACER("CheckPointFault_write");
SCOREP_USER_REGION("CheckPointFault_write", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
logInfo(rank()) << "Writing fault check point.";
// Create array with all pointers
EPIK_USER_REG(r_write_fault, "checkpoint_write_fault");
SCOREP_USER_REGION_DEFINE(r_write_fault);
EPIK_USER_START(r_write_fault);
SCOREP_USER_REGION_BEGIN(r_write_fault, "checkpoint_write_fault", SCOREP_USER_REGION_TYPE_COMMON);
// Attributes
checkH5Err(H5Awrite(m_h5timestepFault[odd()], H5T_NATIVE_INT, ×tepFault));
// Set memory and file space
hsize_t fStart[2] = {fileOffset(), 0};
hsize_t count[2] = {numSides(), numBndGP()};
hid_t h5memSpace = H5Screate_simple(2, count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, fStart, 0L, count, 0L));
for (unsigned int i = 0; i < NUM_VARIABLES; i++) {
checkH5Err(H5Dwrite(m_h5data[odd()][i], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), data(i)));
}
checkH5Err(H5Sclose(h5memSpace));
EPIK_USER_END(r_write_fault);
SCOREP_USER_REGION_END(r_write_fault);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing fault check point. Done.";
}
void seissol::checkpoint::h5::Fault::load(int ×tepFault)
{
if (numSides() == 0)
return;
logInfo(rank()) << "Loading fault checkpoint";
seissol::checkpoint::CheckPoint::load();
hid_t h5file = open(linkFile());
checkH5Err(h5file);
// Attributes
hid_t h5attr = H5Aopen(h5file, "timestep_fault", H5P_DEFAULT);
checkH5Err(h5attr);
checkH5Err(H5Aread(h5attr, H5T_NATIVE_INT, ×tepFault));
checkH5Err(H5Aclose(h5attr));
// Set the memory space (this is the same for all variables)
hsize_t count[2] = {numSides(), numBndGP()};
hid_t h5memSpace = H5Screate_simple(2, count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
// Offset for the file space
hsize_t fStart[2] = {fileOffset(), 0};
// Read the data
for (unsigned int i = 0; i < NUM_VARIABLES; i++) {
hid_t h5data = H5Dopen(h5file, VAR_NAMES[i], H5P_DEFAULT);
checkH5Err(h5data);
hid_t h5fSpace = H5Dget_space(h5data);
checkH5Err(h5fSpace);
// Read the data
checkH5Err(H5Sselect_hyperslab(h5fSpace, H5S_SELECT_SET, fStart, 0L, count, 0L));
checkH5Err(H5Dread(h5data, H5T_NATIVE_DOUBLE, h5memSpace, h5fSpace,
h5XferList(), data(i)));
checkH5Err(H5Sclose(h5fSpace));
checkH5Err(H5Dclose(h5data));
}
checkH5Err(H5Sclose(h5memSpace));
checkH5Err(H5Fclose(h5file));
}
hid_t seissol::checkpoint::h5::Fault::initFile(int odd, const char* filename)
{
hid_t h5file;
if (loaded()) {
// Open the file
h5file = open(filename, false);
checkH5Err(h5file);
// Fault writer
m_h5timestepFault[odd] = H5Aopen(h5file, "timestep_fault", H5P_DEFAULT);
checkH5Err(m_h5timestepFault[odd]);
// Data
for (unsigned int i = 0; i < NUM_VARIABLES; i++) {
m_h5data[odd][i] = H5Dopen(h5file, VAR_NAMES[i], H5P_DEFAULT);
checkH5Err(m_h5data[odd][i]);
}
} else {
// Create the file
hid_t h5plist = H5Pcreate(H5P_FILE_ACCESS);
checkH5Err(h5plist);
checkH5Err(H5Pset_libver_bounds(h5plist, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST));
#ifdef USE_MPI
checkH5Err(H5Pset_fapl_mpio(h5plist, comm(), MPI_INFO_NULL));
#endif // USE_MPI
h5file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, h5plist);
checkH5Err(h5file);
checkH5Err(H5Pclose(h5plist));
// Create scalar dataspace for attributes
hid_t h5spaceScalar = H5Screate(H5S_SCALAR);
checkH5Err(h5spaceScalar);
// Fault writer
m_h5timestepFault[odd] = H5Acreate(h5file, "timestep_fault",
H5T_STD_I32LE, h5spaceScalar, H5P_DEFAULT, H5P_DEFAULT);
checkH5Err(m_h5timestepFault[odd]);
int t = 0;
checkH5Err(H5Awrite(m_h5timestepFault[odd], H5T_NATIVE_INT, &t));
checkH5Err(H5Sclose(h5spaceScalar));
// Variables
for (unsigned int i = 0; i < NUM_VARIABLES; i++) {
h5plist = H5Pcreate(H5P_DATASET_CREATE);
checkH5Err(h5plist);
checkH5Err(H5Pset_layout(h5plist, H5D_CONTIGUOUS));
checkH5Err(H5Pset_alloc_time(h5plist, H5D_ALLOC_TIME_EARLY));
m_h5data[odd][i] = H5Dcreate(h5file, VAR_NAMES[i], H5T_IEEE_F64LE, m_h5fSpaceData,
H5P_DEFAULT, h5plist, H5P_DEFAULT);
checkH5Err(m_h5data[odd][i]);
checkH5Err(H5Pclose(h5plist));
}
}
return h5file;
}
void seissol::checkpoint::h5::Wavefield::load(double &time, int ×tepWavefield, real* dofs)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::setLoaded();
hid_t h5file = open(linkFile());
checkH5Err(h5file);
// Attributes
hid_t h5attr = H5Aopen(h5file, "time", H5P_DEFAULT);
checkH5Err(h5attr);
checkH5Err(H5Aread(h5attr, H5T_NATIVE_DOUBLE, &time));
checkH5Err(H5Aclose(h5attr));
h5attr = H5Aopen(h5file, "timestep_wavefield", H5P_DEFAULT);
checkH5Err(h5attr);
checkH5Err(H5Aread(h5attr, H5T_NATIVE_INT, ×tepWavefield));
checkH5Err(H5Aclose(h5attr));
// Get dataset
hid_t h5data = H5Dopen(h5file, "values", H5P_DEFAULT);
checkH5Err(h5data);
hid_t h5fSpace = H5Dget_space(h5data);
checkH5Err(h5fSpace);
// Read the data
unsigned int offset = 0;
hsize_t fStart = fileOffset();
hsize_t count = dofsPerIteration();
hid_t h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
for (unsigned int i = 0; i < totalIterations()-1; i++) {
checkH5Err(H5Sselect_hyperslab(h5fSpace, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dread(h5data, H5T_NATIVE_DOUBLE, h5memSpace, h5fSpace,
h5XferList(), &dofs[offset]));
// We are finished in less iterations, read data twice
// so everybody needs the same number of iterations
if (i < iterations()-1) {
fStart += count;
offset += count;
}
}
checkH5Err(H5Sclose(h5memSpace));
// Read reminding data in the last iteration
count = numDofs() - (iterations() - 1) * count;
h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(h5fSpace, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dread(h5data, H5T_NATIVE_DOUBLE, h5memSpace, h5fSpace,
h5XferList(), &dofs[offset]));
checkH5Err(H5Sclose(h5memSpace));
checkH5Err(H5Sclose(h5fSpace));
checkH5Err(H5Dclose(h5data));
checkH5Err(H5Fclose(h5file));
}
hid_t seissol::checkpoint::h5::Wavefield::initFile(int odd, const char* filename)
{
hid_t h5file;
if (loaded()) {
// Open the old file
h5file = open(filename, false);
checkH5Err(h5file);
// Time
m_h5time[odd] = H5Aopen(h5file, "time", H5P_DEFAULT);
checkH5Err(m_h5time[odd]);
// Wavefield writer
m_h5timestepWavefield[odd] = H5Aopen(h5file, "timestep_wavefield", H5P_DEFAULT);
checkH5Err(m_h5timestepWavefield[odd]);
// Data
m_h5data[odd] = H5Dopen(h5file, "values", H5P_DEFAULT);
checkH5Err(m_h5data[odd]);
} else {
// Create the file
hid_t h5plist = H5Pcreate(H5P_FILE_ACCESS);
checkH5Err(h5plist);
checkH5Err(H5Pset_libver_bounds(h5plist, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST));
hsize_t align = utils::Env::get<hsize_t>("SEISSOL_CHECKPOINT_ALIGNMENT", 0);
if (align > 0)
checkH5Err(H5Pset_alignment(h5plist, 1, align));
#ifdef USE_MPI
MPIInfo info;
checkH5Err(H5Pset_fapl_mpio(h5plist, seissol::MPI::mpi.comm(), info.get()));
#endif // USE_MPI
h5file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, h5plist);
checkH5Err(h5file);
checkH5Err(H5Pclose(h5plist));
// Create scalar dataspace for attributes
hid_t h5spaceScalar = H5Screate(H5S_SCALAR);
checkH5Err(h5spaceScalar);
// Time
m_h5time[odd] = H5Acreate(h5file, "time", H5T_IEEE_F64LE, h5spaceScalar,
H5P_DEFAULT, H5P_DEFAULT);
checkH5Err(m_h5time[odd]);
// Partitions
hid_t h5partitions = H5Acreate(h5file, "partitions", H5T_STD_I32LE, h5spaceScalar,
H5P_DEFAULT, H5P_DEFAULT);
checkH5Err(h5partitions);
int p = partitions();
checkH5Err(H5Awrite(h5partitions, H5T_NATIVE_INT, &p));
checkH5Err(H5Aclose(h5partitions));
// Wavefield writer
m_h5timestepWavefield[odd] = H5Acreate(h5file, "timestep_wavefield",
H5T_STD_I32LE, h5spaceScalar, H5P_DEFAULT, H5P_DEFAULT);
checkH5Err(m_h5timestepWavefield[odd]);
int t = 0;
checkH5Err(H5Awrite(m_h5timestepWavefield[odd], H5T_NATIVE_INT, &t));
checkH5Err(H5Sclose(h5spaceScalar));
// Variable
h5plist = H5Pcreate(H5P_DATASET_CREATE);
checkH5Err(h5plist);
checkH5Err(H5Pset_layout(h5plist, H5D_CONTIGUOUS));
checkH5Err(H5Pset_alloc_time(h5plist, H5D_ALLOC_TIME_EARLY));
m_h5data[odd] = H5Dcreate(h5file, "values", H5T_IEEE_F64LE, m_h5fSpaceData,
H5P_DEFAULT, h5plist, H5P_DEFAULT);
checkH5Err(m_h5data[odd]);
checkH5Err(H5Pclose(h5plist));
}
return h5file;
}
void seissol::checkpoint::h5::Wavefield::write(double time, int waveFieldTimeStep)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
EPIK_USER_REG(r_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_header);
EPIK_USER_START(r_header);
SCOREP_USER_REGION_BEGIN(r_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
// Time
checkH5Err(H5Awrite(m_h5time[odd()], H5T_NATIVE_DOUBLE, &time));
// Wavefield writer
checkH5Err(H5Awrite(m_h5timestepWavefield[odd()], H5T_NATIVE_INT, &waveFieldTimeStep));
EPIK_USER_END(r_header);
SCOREP_USER_REGION_END(r_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Write the wave field
unsigned int offset = 0;
hsize_t fStart = fileOffset();
hsize_t count = dofsPerIteration();
hid_t h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
for (unsigned int i = 0; i < totalIterations()-1; i++) {
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &const_cast<real*>(dofs())[offset]));
// We are finished in less iterations, read data twice
// so everybody needs the same number of iterations
if (i < iterations()-1) {
fStart += count;
offset += count;
}
}
checkH5Err(H5Sclose(h5memSpace));
// Save reminding data in the last iteration
count = numDofs() - (iterations() - 1) * count;
h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &dofs()[offset]));
checkH5Err(H5Sclose(h5memSpace));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
