void seissol::checkpoint::sionlib::Wavefield::write(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
int file = open(dataFile(odd()), writeMode());
checkErr(file);
// Write the header
SCOREP_USER_REGION_DEFINE(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(sion_coll_fwrite(header, headerSize, 1, file), 1);
SCOREP_USER_REGION_END(r_write_header);
// Save data
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(sion_coll_fwrite(dofs(), sizeof(real), numDofs(), file), numDofs());
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint(file);
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
// Start at the beginning
checkErr(lseek64(file(), 0, SEEK_SET));
// Write the header
EPIK_USER_REG(r_write_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_write_header);
EPIK_USER_START(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
WavefieldHeader header;
header.time = time;
header.timestepWaveField = timestepWaveField;
writeHeader(file(), header);
EPIK_USER_END(r_write_header);
SCOREP_USER_REGION_END(r_write_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Convert to char* to do pointer arithmetic
const char* buffer = reinterpret_cast<const char*>(dofs());
unsigned long left = numDofs()*sizeof(real);
if (alignment()) {
left = (left + alignment() - 1) / alignment();
left *= alignment();
}
while (left > 0) {
unsigned long written = ::write(file(), buffer, left);
if (written <= 0)
checkErr(written, left);
buffer += written;
left -= written;
}
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
// Skip identifier
checkErr(lseek64(file(), sizeof(unsigned long), SEEK_SET));
// Write the header
EPIK_USER_REG(r_write_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_write_header);
EPIK_USER_START(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(::write(file(), &time, sizeof(time)), sizeof(time));
checkErr(::write(file(), ×tepWaveField, sizeof(timestepWaveField)),
sizeof(timestepWaveField));
EPIK_USER_END(r_write_header);
SCOREP_USER_REGION_END(r_write_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Convert to char* to do pointer arithmetic
const char* buffer = reinterpret_cast<const char*>(dofs());
unsigned long left = numDofs()*sizeof(real);
while (left > 0) {
unsigned long written = ::write(file(), buffer, left);
if (written <= 0)
checkErr(written, left);
buffer += written;
left -= written;
}
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::checkpoint::mpio::WavefieldAsync::writePrepare(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_writePrepare");
SCOREP_USER_REGION("CheckPoint_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION);
// Write the header
writeHeader(time, timestepWaveField);
// Create copy of the dofs
memcpy(m_dofsCopy, dofs(), numDofs()*sizeof(real));
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_begin_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all_begin(file(), m_dofsCopy, numDofs(), MPI_DOUBLE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
m_started = true;
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::FaultAsync::writePrepare(int timestepFault)
{
EPIK_TRACER("CheckPointFault_writePrepare");
SCOREP_USER_REGION("CheckPointFault_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
// Write the header
writeHeader(timestepFault);
// Create copy of the data
for (unsigned int i = 0; i < NUM_VARIABLES; i++)
memcpy(&m_dataCopy[i*numSides()*numBndGP()],
data(i), numSides()*numBndGP()*sizeof(double));
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_fault");
SCOREP_USER_REGION_DEFINE(r_write_fault);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_fault, "checkpoint_write_begin_fault", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all_begin(file(), m_dataCopy, numSides() * numBndGP() * NUM_VARIABLES, MPI_DOUBLE));
EPIK_USER_END(r_write_fault);
SCOREP_USER_REGION_END(r_write_fault);
m_started = true;
logInfo(rank()) << "Writing fault check point. Done.";
}
void seissol::checkpoint::mpio::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
// Write the header
writeHeader(time, timestepWaveField);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all(file(), dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::checkpoint::mpio::Wavefield::write(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
// Write the header
writeHeader(header, headerSize);
// Save data
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
unsigned int totalIter = totalIterations();
unsigned int iter = iterations();
unsigned int count = dofsPerIteration();
if (m_useLargeBuffer) {
totalIter = (totalIter + sizeof(real) - 1) / sizeof(real);
iter = (iter + sizeof(real) - 1) / sizeof(real);
count *= sizeof(real);
}
unsigned long offset = 0;
for (unsigned int i = 0; i < totalIter; i++) {
if (i == iter-1)
// Last iteration
count = numDofs() - (iter-1) * count;
checkMPIErr(MPI_File_write_all(file(), const_cast<real*>(&dofs()[offset]), count, MPI_DOUBLE, MPI_STATUS_IGNORE));
if (i < iter-1)
offset += count;
// otherwise we just continue writing the last chunk over and over
else if (i != totalIter-1)
checkMPIErr(MPI_File_seek(file(), -count * sizeof(real), MPI_SEEK_CUR));
}
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::h5::Fault::write(int timestepFault)
{
EPIK_TRACER("CheckPointFault_write");
SCOREP_USER_REGION("CheckPointFault_write", SCOREP_USER_REGION_TYPE_FUNCTION);
if (numSides() == 0)
return;
logInfo(rank()) << "Writing fault check point.";
// Create array with all pointers
EPIK_USER_REG(r_write_fault, "checkpoint_write_fault");
SCOREP_USER_REGION_DEFINE(r_write_fault);
EPIK_USER_START(r_write_fault);
SCOREP_USER_REGION_BEGIN(r_write_fault, "checkpoint_write_fault", SCOREP_USER_REGION_TYPE_COMMON);
// Attributes
checkH5Err(H5Awrite(m_h5timestepFault[odd()], H5T_NATIVE_INT, ×tepFault));
// Set memory and file space
hsize_t fStart[2] = {fileOffset(), 0};
hsize_t count[2] = {numSides(), numBndGP()};
hid_t h5memSpace = H5Screate_simple(2, count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, fStart, 0L, count, 0L));
for (unsigned int i = 0; i < NUM_VARIABLES; i++) {
checkH5Err(H5Dwrite(m_h5data[odd()][i], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), data(i)));
}
checkH5Err(H5Sclose(h5memSpace));
EPIK_USER_END(r_write_fault);
SCOREP_USER_REGION_END(r_write_fault);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing fault check point. Done.";
}
int compute_solution(int nprocs, int myrank, const int max_iters, int nintci, int nintcf, int nextcf, int** lcc, double* bp,
double* bs, double* bw, double* bl, double* bn, double* be, double* bh,
double* cnorm, double* var, double *su, double* cgup, double* residual_ratio,
int* local_global_index, int* global_local_index, int nghb_cnt,
int* nghb_to_rank, int* send_cnt, int** send_lst, int *recv_cnt, int** recv_lst){
// Add SCOREP manual instrumentation
#ifdef SCOREP
SCOREP_USER_REGION_DEFINE(handle1);
SCOREP_USER_REGION_DEFINE(handle2);
SCOREP_USER_REGION_DEFINE(handle3);
SCOREP_USER_REGION_DEFINE(handle4);
SCOREP_USER_REGION_DEFINE(handle5);
SCOREP_USER_REGION_DEFINE(handle6);
SCOREP_USER_REGION_DEFINE(handle7);
SCOREP_USER_REGION_DEFINE(handle8);
SCOREP_USER_REGION_DEFINE(handle9);
SCOREP_USER_REGION_DEFINE(handle10);
SCOREP_USER_REGION_DEFINE(handle_break);
#endif
#ifdef SCOREP
SCOREP_USER_REGION_BEGIN( handle1, "handle1 - Initialization of variables and reference residuals.",SCOREP_USER_REGION_TYPE_COMMON );
#endif
/** parameters used in gccg */
int iter = 1;
int if1 = 0;
int if2 = 0;
int nor = 1;
int nor1 = nor - 1;
int nc = 0;
int nomax = 3;
/** the reference residual */
double resref = 0.0;
/** array storing residuals */
double *resvec = (double *) calloc(sizeof(double), (nintcf + 1));
// initialize the reference residual
for ( nc = nintci; nc <= nintcf; nc++ ) {
resvec[nc] = su[nc];
resref = resref + resvec[nc] * resvec[nc];
}
#ifdef SCOREP
SCOREP_USER_REGION_END( handle1 );
SCOREP_USER_REGION_BEGIN( handle2, "handle2 - 1st Allreduce.",SCOREP_USER_REGION_TYPE_COMMON );
#endif
// A2.3
double global_resref = 0;
MPI_Allreduce(&resref, &global_resref, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
resref = global_resref;
#ifdef SCOREP
SCOREP_USER_REGION_END( handle2 );
SCOREP_USER_REGION_BEGIN( handle3, "handle3 - Calculation of the residue sum.",SCOREP_USER_REGION_TYPE_COMMON );
#endif
resref = sqrt(resref);
if ( resref < 1.0e-15 ) {
fprintf(stderr, "Residue sum less than 1.e-15 - %lf\n", resref);
return 0;
}
#ifdef SCOREP
SCOREP_USER_REGION_END( handle3 );
SCOREP_USER_REGION_BEGIN( handle4, "handle4 - Memory allocation.",SCOREP_USER_REGION_TYPE_COMMON );
#endif
// Counting the number of ghost cells to extend the direc1
int ghost_cells_recv = 0, ghost_cells_send = 0;
int proc, i, j;
for (proc = 0; proc < nghb_cnt; proc++) {
ghost_cells_recv += recv_cnt[proc];
ghost_cells_send += send_cnt[proc];
}
/** the computation vectors */
// TODO:
double *direc1 = (double *) calloc(sizeof(double), ((nextcf + 1) + ghost_cells_recv));
double *direc2 = (double *) calloc(sizeof(double), (nextcf + 1));
double *adxor1 = (double *) calloc(sizeof(double), (nintcf + 1));
double *adxor2 = (double *) calloc(sizeof(double), (nintcf + 1));
double *dxor1 = (double *) calloc(sizeof(double), (nintcf + 1));
double *dxor2 = (double *) calloc(sizeof(double), (nintcf + 1));
// Determine displacements for sending
int **displacements = (int **) malloc(sizeof(double)*nghb_cnt);
int **blocklenghts = (int **) malloc(sizeof(double)*nghb_cnt);
for(proc = 0; proc < nghb_cnt; proc++) {
displacements[proc] = (int*)calloc(send_cnt[proc],sizeof(int));
blocklenghts[proc] = (int*)calloc(send_cnt[proc],sizeof(int));
}
j = 0;
for (proc = 0; proc < nghb_cnt; proc++) {
for (i = 0; i < send_cnt[proc]; i++) {
displacements[proc][i] = global_local_index[send_lst[proc][i]];
blocklenghts[proc][i] = 1;
}
}
MPI_Request request;
MPI_Datatype *indextype;
indextype = (MPI_Datatype *) malloc(sizeof(*indextype)*nghb_cnt);
for (proc = 0; proc < nghb_cnt; proc++) {
MPI_Type_indexed(send_cnt[proc], blocklenghts[proc], displacements[proc], MPI_DOUBLE, &(indextype[proc]));
MPI_Type_commit(&(indextype[proc]));
}
#ifdef SCOREP
SCOREP_USER_REGION_END( handle4 );
SCOREP_USER_REGION_BEGIN( handle5, "handle5 - Computation phase1. direc1 update.",SCOREP_USER_REGION_TYPE_COMMON );
#endif
while ( iter < max_iters ) {
/********** START COMP PHASE 1 **********/
// update the old values of direc
for ( nc = nintci; nc <= nintcf; nc++ ) {
direc1[nc] = direc1[nc] + resvec[nc] * cgup[nc];
}
#ifdef SCOREP
SCOREP_USER_REGION_END( handle5 );
SCOREP_USER_REGION_BEGIN( handle6, "handle6 - Computation phase1. direc1 communication",SCOREP_USER_REGION_TYPE_COMMON );
#endif
// Communication of direc1 - start
for (proc = 0; proc < nghb_cnt; proc++) {
MPI_Isend(direc1, 1, indextype[proc], nghb_to_rank[proc], 0, MPI_COMM_WORLD, &request);
}
// Reference position in the direc1
int ref_pos = nextcf + 1;
for (proc = 0; proc < nghb_cnt; proc++) {
MPI_Recv(&(direc1[ref_pos]), recv_cnt[proc], MPI_DOUBLE, nghb_to_rank[proc], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
ref_pos += recv_cnt[proc];
}
// Communication of direc1 - stop
#ifdef SCOREP
SCOREP_USER_REGION_END( handle6 );
SCOREP_USER_REGION_BEGIN( handle7, "handle7 - Computation phase1. direc2 computation",SCOREP_USER_REGION_TYPE_COMMON );
#endif
// compute new guess (approximation) for direc
for ( nc = nintci; nc <= nintcf; nc++ ) {
direc2[nc] = bp[nc] * direc1[nc] - bs[nc] * direc1[global_local_index[lcc[nc][0]]]
- be[nc] * direc1[global_local_index[lcc[nc][1]]] - bn[nc] * direc1[global_local_index[lcc[nc][2]]]
- bw[nc] * direc1[global_local_index[lcc[nc][3]]] - bl[nc] * direc1[global_local_index[lcc[nc][4]]]
- bh[nc] * direc1[global_local_index[lcc[nc][5]]];
}
/********** END COMP PHASE 1 **********/
#ifdef SCOREP
SCOREP_USER_REGION_END( handle7 );
SCOREP_USER_REGION_BEGIN( handle8, "handle8 - Computation phase2. occ computation",SCOREP_USER_REGION_TYPE_COMMON );
#endif
/********** START COMP PHASE 2 **********/
// execute normalization steps
double oc1, oc2, occ;
if ( nor1 == 1 ) {
oc1 = 0;
occ = 0;
for ( nc = nintci; nc <= nintcf; nc++ ) {
occ = occ + direc2[nc] * adxor1[nc];
}
// A2.3
double global_occ = 0.0;
MPI_Allreduce(&occ, &global_occ, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
occ = global_occ;
oc1 = occ / cnorm[1];
for ( nc = nintci; nc <= nintcf; nc++ ) {
direc2[nc] = direc2[nc] - oc1 * adxor1[nc];
direc1[nc] = direc1[nc] - oc1 * dxor1[nc];
}
if1++;
} else {
if ( nor1 == 2 ) {
oc1 = 0;
occ = 0;
for ( nc = nintci; nc <= nintcf; nc++ ) {
occ = occ + direc2[nc] * adxor1[nc];
}
// A2.3
double global_occ = 0.0;
MPI_Allreduce(&occ, &global_occ, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
occ = global_occ;
oc1 = occ / cnorm[1];
oc2 = 0;
occ = 0;
for ( nc = nintci; nc <= nintcf; nc++ ) {
occ = occ + direc2[nc] * adxor2[nc];
}
// A2.3
MPI_Allreduce(&occ, &global_occ, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
occ = global_occ;
oc2 = occ / cnorm[2];
for ( nc = nintci; nc <= nintcf; nc++ ) {
direc1[nc] = direc1[nc] - oc1 * dxor1[nc] - oc2 * dxor2[nc];
direc2[nc] = direc2[nc] - oc1 * adxor1[nc] - oc2 * adxor2[nc];
}
if2++;
}
}
#ifdef SCOREP
SCOREP_USER_REGION_END( handle8 );
SCOREP_USER_REGION_BEGIN( handle9, "handle9 - Computation phase2. residual_ratio computation - before break",SCOREP_USER_REGION_TYPE_COMMON );
#endif
// compute the new residual
cnorm[nor] = 0;
double omega = 0;
for ( nc = nintci; nc <= nintcf; nc++ ) {
cnorm[nor] = cnorm[nor] + direc2[nc] * direc2[nc];
omega = omega + resvec[nc] * direc2[nc];
}
// A2.3
double global_cnorm_nor = 0.0, global_omega = 0.0;
MPI_Allreduce(&(cnorm[nor]), &global_cnorm_nor, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&omega, &global_omega, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
cnorm[nor] = global_cnorm_nor;
omega = global_omega;
omega = omega / cnorm[nor];
double res_updated = 0.0;
for ( nc = nintci; nc <= nintcf; nc++ ) {
resvec[nc] = resvec[nc] - omega * direc2[nc];
res_updated = res_updated + resvec[nc] * resvec[nc];
var[nc] = var[nc] + omega * direc1[nc];
}
// A2.3
double global_res_updated = 0.0;
MPI_Allreduce(&res_updated, &global_res_updated, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
res_updated = global_res_updated;
res_updated = sqrt(res_updated);
*residual_ratio = res_updated / resref;
#ifdef SCOREP
SCOREP_USER_REGION_END( handle9 );
#endif
// exit on no improvements of residual
if ( *residual_ratio <= 1.0e-10 ) break;
#ifdef SCOREP
SCOREP_USER_REGION_BEGIN( handle_break, "handle9 - Computation phase2. residual_ratio computation - after break",SCOREP_USER_REGION_TYPE_COMMON );
#endif
iter++;
// prepare additional arrays for the next iteration step
if ( nor == nomax ) {
nor = 1;
} else {
if ( nor == 1 ) {
for ( nc = nintci; nc <= nintcf; nc++ ) {
dxor1[nc] = direc1[nc];
adxor1[nc] = direc2[nc];
}
} else {
if ( nor == 2 ) {
for ( nc = nintci; nc <= nintcf; nc++ ) {
dxor2[nc] = direc1[nc];
adxor2[nc] = direc2[nc];
}
}
}
nor++;
}
nor1 = nor - 1;
/********** END COMP PHASE 2 **********/
}
#ifdef SCOREP
SCOREP_USER_REGION_END( handle_break );
SCOREP_USER_REGION_BEGIN( handle10, "handle10 - Memory freeing",SCOREP_USER_REGION_TYPE_COMMON );
#endif
for (i = 0; i < nghb_cnt; i++){
free(displacements[i]);
}
free(displacements);
free(indextype);
free(direc1);
free(direc2);
free(adxor1);
free(adxor2);
free(dxor1);
free(dxor2);
free(resvec);
return iter;
#ifdef SCOREP
SCOREP_USER_REGION_END( handle10 );
#endif
}
void seissol::checkpoint::h5::Wavefield::write(double time, int waveFieldTimeStep)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
EPIK_USER_REG(r_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_header);
EPIK_USER_START(r_header);
SCOREP_USER_REGION_BEGIN(r_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
// Time
checkH5Err(H5Awrite(m_h5time[odd()], H5T_NATIVE_DOUBLE, &time));
// Wavefield writer
checkH5Err(H5Awrite(m_h5timestepWavefield[odd()], H5T_NATIVE_INT, &waveFieldTimeStep));
EPIK_USER_END(r_header);
SCOREP_USER_REGION_END(r_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Write the wave field
unsigned int offset = 0;
hsize_t fStart = fileOffset();
hsize_t count = dofsPerIteration();
hid_t h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
for (unsigned int i = 0; i < totalIterations()-1; i++) {
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &const_cast<real*>(dofs())[offset]));
// We are finished in less iterations, read data twice
// so everybody needs the same number of iterations
if (i < iterations()-1) {
fStart += count;
offset += count;
}
}
checkH5Err(H5Sclose(h5memSpace));
// Save reminding data in the last iteration
count = numDofs() - (iterations() - 1) * count;
h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &dofs()[offset]));
checkH5Err(H5Sclose(h5memSpace));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
