void seissol::checkpoint::sionlib::Wavefield::write(const void* header, size_t headerSize) { SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Checkpoint backend: Writing."; int file = open(dataFile(odd()), writeMode()); checkErr(file); // Write the header SCOREP_USER_REGION_DEFINE(r_write_header); SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON); checkErr(sion_coll_fwrite(header, headerSize, 1, file), 1); SCOREP_USER_REGION_END(r_write_header); // Save data SCOREP_USER_REGION_DEFINE(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); checkErr(sion_coll_fwrite(dofs(), sizeof(real), numDofs(), file), numDofs()); SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(file); logInfo(rank()) << "Checkpoint backend: Writing. Done."; }
void seissol::checkpoint::mpio::WavefieldAsync::writePrepare(double time, int timestepWaveField) { EPIK_TRACER("CheckPoint_writePrepare"); SCOREP_USER_REGION("CheckPoint_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION); // Write the header writeHeader(time, timestepWaveField); // Create copy of the dofs memcpy(m_dofsCopy, dofs(), numDofs()*sizeof(real)); // Save data EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_wavefield"); SCOREP_USER_REGION_DEFINE(r_write_wavefield); EPIK_USER_START(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_begin_wavefield", SCOREP_USER_REGION_TYPE_COMMON); checkMPIErr(setDataView(file())); checkMPIErr(MPI_File_write_all_begin(file(), m_dofsCopy, numDofs(), MPI_DOUBLE)); EPIK_USER_END(r_write_wavefield); SCOREP_USER_REGION_END(r_write_wavefield); m_started = true; logInfo(rank()) << "Checkpoint backend: Writing. Done."; }
void seissol::checkpoint::mpio::Wavefield::write(double time, int timestepWaveField) { EPIK_TRACER("CheckPoint_write"); SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Writing check point."; // Write the header writeHeader(time, timestepWaveField); // Save data EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield"); SCOREP_USER_REGION_DEFINE(r_write_wavefield); EPIK_USER_START(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); checkMPIErr(setDataView(file())); checkMPIErr(MPI_File_write_all(file(), dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE)); EPIK_USER_END(r_write_wavefield); SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(); logInfo(rank()) << "Writing check point. Done."; }
void seissol::checkpoint::mpio::Wavefield::load(real* dofs) { logInfo(rank()) << "Loading wave field checkpoint"; seissol::checkpoint::CheckPoint::setLoaded(); MPI_File file = open(); if (file == MPI_FILE_NULL) logError() << "Could not open checkpoint file"; // Read and broadcast header checkMPIErr(setHeaderView(file)); if (rank() == 0) checkMPIErr(MPI_File_read(file, header().data(), 1, headerType(), MPI_STATUS_IGNORE)); MPI_Bcast(header().data(), 1, headerType(), 0, comm()); // Read dofs checkMPIErr(setDataView(file)); checkMPIErr(MPI_File_read_all(file, dofs, numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE)); // Close the file checkMPIErr(MPI_File_close(&file)); }
void seissol::checkpoint::mpio::Wavefield::load(double &time, int ×tepWaveField) { logInfo(rank()) << "Loading wave field checkpoint"; seissol::checkpoint::CheckPoint::load(); MPI_File file = open(); if (file == MPI_FILE_NULL) logError() << "Could not open checkpoint file"; // Read and broadcast header checkMPIErr(setHeaderView(file)); Header header; if (rank() == 0) checkMPIErr(MPI_File_read(file, &header, 1, headerType(), MPI_STATUS_IGNORE)); MPI_Bcast(&header, 1, headerType(), 0, comm()); time = header.time; timestepWaveField = header.timestepWavefield; // Read dofs checkMPIErr(setDataView(file)); checkMPIErr(MPI_File_read_all(file, dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE)); // Close the file checkMPIErr(MPI_File_close(&file)); }
void seissol::checkpoint::posix::Wavefield::load(double &time, int ×tepWaveField, real* dofs) { logInfo(rank()) << "Loading wave field checkpoint"; seissol::checkpoint::CheckPoint::setLoaded(); int file = open(); checkErr(file); // Read header WavefieldHeader header; readHeader(file, header); time = header.time; timestepWaveField = header.timestepWaveField; // Skip other processes before this in the group checkErr(lseek64(file, groupOffset() * sizeof(real), SEEK_CUR)); // Convert to char* to do pointer arithmetic char* buffer = reinterpret_cast<char*>(dofs); unsigned long left = numDofs()*sizeof(real); // Read dofs while (left > 0) { unsigned long readSize = read(file, buffer, left); if (readSize <= 0) checkErr(readSize, left); buffer += readSize; left -= readSize; } // Close the file checkErr(::close(file)); }
void seissol::checkpoint::posix::Wavefield::load(double &time, int ×tepWaveField) { logInfo(rank()) << "Loading wave field checkpoint"; seissol::checkpoint::CheckPoint::load(); int file = open(); checkErr(file); // Skip identifier checkErr(lseek64(file, sizeof(unsigned long), SEEK_SET)); // Read header checkErr(read(file, &time, sizeof(time)), sizeof(time)); checkErr(read(file, ×tepWaveField, sizeof(timestepWaveField)), sizeof(timestepWaveField)); // Convert to char* to do pointer arithmetic char* buffer = reinterpret_cast<char*>(dofs()); unsigned long left = numDofs()*sizeof(real); // Read dofs while (left > 0) { unsigned long readSize = read(file, buffer, left); if (readSize <= 0) checkErr(readSize, left); buffer += readSize; left -= readSize; } // Close the file checkErr(::close(file)); }
void seissol::checkpoint::sionlib::Wavefield::load(real* dofs) { logInfo(rank()) << "Loading wave field checkpoint"; seissol::checkpoint::CheckPoint::setLoaded(); int file = open(linkFile(), readMode()); checkErr(file); // Read header checkErr(sion_coll_fread(header().data(), header().size(), 1, file), 1); // Read dofs checkErr(sion_coll_fread(dofs, sizeof(double), numDofs(), file), numDofs()); // Close the file sionClose(file); }
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField) { EPIK_TRACER("CheckPoint_write"); SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Checkpoint backend: Writing."; // Start at the beginning checkErr(lseek64(file(), 0, SEEK_SET)); // Write the header EPIK_USER_REG(r_write_header, "checkpoint_write_header"); SCOREP_USER_REGION_DEFINE(r_write_header); EPIK_USER_START(r_write_header); SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON); WavefieldHeader header; header.time = time; header.timestepWaveField = timestepWaveField; writeHeader(file(), header); EPIK_USER_END(r_write_header); SCOREP_USER_REGION_END(r_write_header); // Save data EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield"); SCOREP_USER_REGION_DEFINE(r_write_wavefield); EPIK_USER_START(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); // Convert to char* to do pointer arithmetic const char* buffer = reinterpret_cast<const char*>(dofs()); unsigned long left = numDofs()*sizeof(real); if (alignment()) { left = (left + alignment() - 1) / alignment(); left *= alignment(); } while (left > 0) { unsigned long written = ::write(file(), buffer, left); if (written <= 0) checkErr(written, left); buffer += written; left -= written; } EPIK_USER_END(r_write_wavefield); SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(); logInfo(rank()) << "Checkpoint backend: Writing. Done."; }
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField) { EPIK_TRACER("CheckPoint_write"); SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Writing check point."; // Skip identifier checkErr(lseek64(file(), sizeof(unsigned long), SEEK_SET)); // Write the header EPIK_USER_REG(r_write_header, "checkpoint_write_header"); SCOREP_USER_REGION_DEFINE(r_write_header); EPIK_USER_START(r_write_header); SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON); checkErr(::write(file(), &time, sizeof(time)), sizeof(time)); checkErr(::write(file(), ×tepWaveField, sizeof(timestepWaveField)), sizeof(timestepWaveField)); EPIK_USER_END(r_write_header); SCOREP_USER_REGION_END(r_write_header); // Save data EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield"); SCOREP_USER_REGION_DEFINE(r_write_wavefield); EPIK_USER_START(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); // Convert to char* to do pointer arithmetic const char* buffer = reinterpret_cast<const char*>(dofs()); unsigned long left = numDofs()*sizeof(real); while (left > 0) { unsigned long written = ::write(file(), buffer, left); if (written <= 0) checkErr(written, left); buffer += written; left -= written; } EPIK_USER_END(r_write_wavefield); SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(); logInfo(rank()) << "Writing check point. Done."; }
void seissol::checkpoint::mpio::Wavefield::write(const void* header, size_t headerSize) { SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Checkpoint backend: Writing."; // Write the header writeHeader(header, headerSize); // Save data SCOREP_USER_REGION_DEFINE(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); checkMPIErr(setDataView(file())); unsigned int totalIter = totalIterations(); unsigned int iter = iterations(); unsigned int count = dofsPerIteration(); if (m_useLargeBuffer) { totalIter = (totalIter + sizeof(real) - 1) / sizeof(real); iter = (iter + sizeof(real) - 1) / sizeof(real); count *= sizeof(real); } unsigned long offset = 0; for (unsigned int i = 0; i < totalIter; i++) { if (i == iter-1) // Last iteration count = numDofs() - (iter-1) * count; checkMPIErr(MPI_File_write_all(file(), const_cast<real*>(&dofs()[offset]), count, MPI_DOUBLE, MPI_STATUS_IGNORE)); if (i < iter-1) offset += count; // otherwise we just continue writing the last chunk over and over else if (i != totalIter-1) checkMPIErr(MPI_File_seek(file(), -count * sizeof(real), MPI_SEEK_CUR)); } SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(); logInfo(rank()) << "Checkpoint backend: Writing. Done."; }
void seissol::checkpoint::h5::Wavefield::load(double &time, int ×tepWavefield, real* dofs) { logInfo(rank()) << "Loading wave field checkpoint"; seissol::checkpoint::CheckPoint::setLoaded(); hid_t h5file = open(linkFile()); checkH5Err(h5file); // Attributes hid_t h5attr = H5Aopen(h5file, "time", H5P_DEFAULT); checkH5Err(h5attr); checkH5Err(H5Aread(h5attr, H5T_NATIVE_DOUBLE, &time)); checkH5Err(H5Aclose(h5attr)); h5attr = H5Aopen(h5file, "timestep_wavefield", H5P_DEFAULT); checkH5Err(h5attr); checkH5Err(H5Aread(h5attr, H5T_NATIVE_INT, ×tepWavefield)); checkH5Err(H5Aclose(h5attr)); // Get dataset hid_t h5data = H5Dopen(h5file, "values", H5P_DEFAULT); checkH5Err(h5data); hid_t h5fSpace = H5Dget_space(h5data); checkH5Err(h5fSpace); // Read the data unsigned int offset = 0; hsize_t fStart = fileOffset(); hsize_t count = dofsPerIteration(); hid_t h5memSpace = H5Screate_simple(1, &count, 0L); checkH5Err(h5memSpace); checkH5Err(H5Sselect_all(h5memSpace)); for (unsigned int i = 0; i < totalIterations()-1; i++) { checkH5Err(H5Sselect_hyperslab(h5fSpace, H5S_SELECT_SET, &fStart, 0L, &count, 0L)); checkH5Err(H5Dread(h5data, H5T_NATIVE_DOUBLE, h5memSpace, h5fSpace, h5XferList(), &dofs[offset])); // We are finished in less iterations, read data twice // so everybody needs the same number of iterations if (i < iterations()-1) { fStart += count; offset += count; } } checkH5Err(H5Sclose(h5memSpace)); // Read reminding data in the last iteration count = numDofs() - (iterations() - 1) * count; h5memSpace = H5Screate_simple(1, &count, 0L); checkH5Err(h5memSpace); checkH5Err(H5Sselect_all(h5memSpace)); checkH5Err(H5Sselect_hyperslab(h5fSpace, H5S_SELECT_SET, &fStart, 0L, &count, 0L)); checkH5Err(H5Dread(h5data, H5T_NATIVE_DOUBLE, h5memSpace, h5fSpace, h5XferList(), &dofs[offset])); checkH5Err(H5Sclose(h5memSpace)); checkH5Err(H5Sclose(h5fSpace)); checkH5Err(H5Dclose(h5data)); checkH5Err(H5Fclose(h5file)); }
void seissol::checkpoint::h5::Wavefield::write(double time, int waveFieldTimeStep) { EPIK_TRACER("CheckPoint_write"); SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Writing check point."; EPIK_USER_REG(r_header, "checkpoint_write_header"); SCOREP_USER_REGION_DEFINE(r_header); EPIK_USER_START(r_header); SCOREP_USER_REGION_BEGIN(r_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON); // Time checkH5Err(H5Awrite(m_h5time[odd()], H5T_NATIVE_DOUBLE, &time)); // Wavefield writer checkH5Err(H5Awrite(m_h5timestepWavefield[odd()], H5T_NATIVE_INT, &waveFieldTimeStep)); EPIK_USER_END(r_header); SCOREP_USER_REGION_END(r_header); // Save data EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield"); SCOREP_USER_REGION_DEFINE(r_write_wavefield); EPIK_USER_START(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); // Write the wave field unsigned int offset = 0; hsize_t fStart = fileOffset(); hsize_t count = dofsPerIteration(); hid_t h5memSpace = H5Screate_simple(1, &count, 0L); checkH5Err(h5memSpace); checkH5Err(H5Sselect_all(h5memSpace)); for (unsigned int i = 0; i < totalIterations()-1; i++) { checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L)); checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData, h5XferList(), &const_cast<real*>(dofs())[offset])); // We are finished in less iterations, read data twice // so everybody needs the same number of iterations if (i < iterations()-1) { fStart += count; offset += count; } } checkH5Err(H5Sclose(h5memSpace)); // Save reminding data in the last iteration count = numDofs() - (iterations() - 1) * count; h5memSpace = H5Screate_simple(1, &count, 0L); checkH5Err(h5memSpace); checkH5Err(H5Sselect_all(h5memSpace)); checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L)); checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData, h5XferList(), &dofs()[offset])); checkH5Err(H5Sclose(h5memSpace)); EPIK_USER_END(r_write_wavefield); SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(); logInfo(rank()) << "Writing check point. Done."; }
void AmalgamationInfo<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::UnamalgamateAggregates(const Aggregates& aggregates, Teuchos::ArrayRCP<LocalOrdinal>& aggStart, Teuchos::ArrayRCP<GlobalOrdinal>& aggToRowMap) const { int myPid = aggregates.GetMap()->getComm()->getRank(); Teuchos::ArrayView<const GO> nodeGlobalElts = aggregates.GetMap()->getNodeElementList(); Teuchos::ArrayRCP<LO> procWinner = aggregates.GetProcWinner()->getDataNonConst(0); Teuchos::ArrayRCP<LO> vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0); LO size = procWinner.size(); GO numAggregates = aggregates.GetNumAggregates(); std::vector<LO> sizes(numAggregates); if (stridedblocksize_ == 1) { for (LO lnode = 0; lnode < size; ++lnode) { LO myAgg = vertex2AggId[lnode]; if (procWinner[lnode] == myPid) sizes[myAgg] += 1; } } else { for (LO lnode = 0; lnode < size; ++lnode) { LO myAgg = vertex2AggId[lnode]; if (procWinner[lnode] == myPid) { GO gnodeid = nodeGlobalElts[lnode]; for (LocalOrdinal k = 0; k < stridedblocksize_; k++) { GlobalOrdinal gDofIndex = ComputeGlobalDOF(gnodeid,k); if (columnMap_->isNodeGlobalElement(gDofIndex)) sizes[myAgg] += 1; } } } } aggStart = ArrayRCP<LO>(numAggregates+1,0); aggStart[0]=0; for (GO i=0; i<numAggregates; ++i) { aggStart[i+1] = aggStart[i] + sizes[i]; } aggToRowMap = ArrayRCP<GO>(aggStart[numAggregates],0); // count, how many dofs have been recorded for each aggregate so far Array<LO> numDofs(numAggregates, 0); // empty array with number of Dofs for each aggregate if (stridedblocksize_ == 1) { for (LO lnode = 0; lnode < size; ++lnode) { LO myAgg = vertex2AggId[lnode]; if (procWinner[lnode] == myPid) { aggToRowMap[ aggStart[myAgg] + numDofs[myAgg] ] = ComputeGlobalDOF(nodeGlobalElts[lnode]); ++(numDofs[myAgg]); } } } else { for (LO lnode = 0; lnode < size; ++lnode) { LO myAgg = vertex2AggId[lnode]; if (procWinner[lnode] == myPid) { GO gnodeid = nodeGlobalElts[lnode]; for (LocalOrdinal k = 0; k < stridedblocksize_; k++) { GlobalOrdinal gDofIndex = ComputeGlobalDOF(gnodeid,k); if (columnMap_->isNodeGlobalElement(gDofIndex)) { aggToRowMap[ aggStart[myAgg] + numDofs[myAgg] ] = gDofIndex; ++(numDofs[myAgg]); } } } } } // todo plausibility check: entry numDofs[k] == aggToRowMap[k].size() } //UnamalgamateAggregates
void AmalgamationInfo<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::UnamalgamateAggregatesLO(const Aggregates& aggregates, Teuchos::ArrayRCP<LO>& aggStart, Teuchos::ArrayRCP<LO>& aggToRowMap) const { int myPid = aggregates.GetMap()->getComm()->getRank(); Teuchos::ArrayView<const GO> nodeGlobalElts = aggregates.GetMap()->getNodeElementList(); Teuchos::ArrayRCP<LO> procWinner = aggregates.GetProcWinner() ->getDataNonConst(0); Teuchos::ArrayRCP<LO> vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0); const GO numAggregates = aggregates.GetNumAggregates(); // FIXME: Do we need to compute size here? Or can we use existing? LO size = procWinner.size(); std::vector<LO> sizes(numAggregates); if (stridedblocksize_ == 1) { for (LO lnode = 0; lnode < size; lnode++) if (procWinner[lnode] == myPid) sizes[vertex2AggId[lnode]]++; } else { for (LO lnode = 0; lnode < size; lnode++) if (procWinner[lnode] == myPid) { GO nodeGID = nodeGlobalElts[lnode]; for (LO k = 0; k < stridedblocksize_; k++) { GO GID = ComputeGlobalDOF(nodeGID, k); if (columnMap_->isNodeGlobalElement(GID)) sizes[vertex2AggId[lnode]]++; } } } aggStart = ArrayRCP<LO>(numAggregates+1); // FIXME: useless initialization with zeros aggStart[0] = 0; for (GO i = 0; i < numAggregates; i++) aggStart[i+1] = aggStart[i] + sizes[i]; aggToRowMap = ArrayRCP<LO>(aggStart[numAggregates], 0); // count, how many dofs have been recorded for each aggregate so far Array<LO> numDofs(numAggregates, 0); // empty array with number of DOFs for each aggregate if (stridedblocksize_ == 1) { for (LO lnode = 0; lnode < size; ++lnode) if (procWinner[lnode] == myPid) { LO myAgg = vertex2AggId[lnode]; aggToRowMap[aggStart[myAgg] + numDofs[myAgg]] = lnode; numDofs[myAgg]++; } } else { for (LO lnode = 0; lnode < size; ++lnode) if (procWinner[lnode] == myPid) { LO myAgg = vertex2AggId[lnode]; GO nodeGID = nodeGlobalElts[lnode]; for (LO k = 0; k < stridedblocksize_; k++) { GO GID = ComputeGlobalDOF(nodeGID, k); if (columnMap_->isNodeGlobalElement(GID)) { aggToRowMap[aggStart[myAgg] + numDofs[myAgg]] = lnode*stridedblocksize_ + k; numDofs[myAgg]++; } } } } // todo plausibility check: entry numDofs[k] == aggToRowMap[k].size() } //UnamalgamateAggregates