void DCDataSet::create(const CollectionType& colType,
hid_t group, const Dimensions size, uint32_t ndims, bool compression)
throw (DCException)
{
log_msg(2, "DCDataSet::create (%s, size %s)", name.c_str(), size.toString().c_str());
if (opened)
throw DCException(getExceptionString("create: dataset is already open"));
// if the dataset already exists, remove/unlink it
// note that this won't free the memory occupied by this
// dataset, however, there currently is no function to delete
// a dataset
if (!checkExistence || (checkExistence && H5Lexists(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT)))
H5Ldelete(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT);
this->ndims = ndims;
this->compression = compression;
this->datatype = colType.getDataType();
getLogicalSize().set(size);
setChunking(colType.getSize());
setCompression();
if (getPhysicalSize().getScalarSize() != 0)
{
hsize_t *max_dims = new hsize_t[ndims];
for (size_t i = 0; i < ndims; ++i)
max_dims[i] = H5F_UNLIMITED;
dataspace = H5Screate_simple(ndims, getPhysicalSize().getPointer(), max_dims);
delete[] max_dims;
max_dims = NULL;
} else
dataspace = H5Screate(H5S_NULL);
if (dataspace < 0)
throw DCException(getExceptionString("create: Failed to create dataspace"));
// create the new dataset
dataset = H5Dcreate(group, this->name.c_str(), this->datatype, dataspace,
H5P_DEFAULT, dsetProperties, H5P_DEFAULT);
if (dataset < 0)
throw DCException(getExceptionString("create: Failed to create dataset"));
isReference = false;
opened = true;
}
void DCDataSet::setCompression()
throw (DCException)
{
if (this->compression && getPhysicalSize().getScalarSize() != 0)
{
// shuffling reorders bytes for better compression
// set gzip compression level (1=lowest - 9=highest)
if (H5Pset_shuffle(this->dsetProperties) < 0 ||
H5Pset_deflate(this->dsetProperties, 1) < 0)
throw DCException(getExceptionString("setCompression: Failed to set compression"));
}
}
void DCDataSet::setChunking(size_t typeSize)
throw (DCException)
{
if (getPhysicalSize().getScalarSize() != 0)
{
// get chunking dimensions
hsize_t chunk_dims[ndims];
DCHelper::getOptimalChunkDims(getPhysicalSize().getPointer(), ndims,
typeSize, chunk_dims);
if (H5Pset_chunk(this->dsetProperties, ndims, chunk_dims) < 0)
{
for (size_t i = 0; i < ndims; ++i)
{
log_msg(1, "chunk_dims[%llu] = %llu",
(long long unsigned) i, (long long unsigned) (chunk_dims[i]));
}
throw DCException(getExceptionString("setChunking: Failed to set chunking"));
}
}
}
bool Chunk::splitIfShould( long dataWritten ){
_dataWritten += dataWritten;
if ( _dataWritten < MaxChunkSize / 5 )
return false;
_dataWritten = 0;
if ( _min.woCompare( _max ) == 0 ){
log() << "SHARD PROBLEM** shard is too big, but can't split: " << toString() << endl;
return false;
}
long size = getPhysicalSize();
if ( size < MaxChunkSize )
return false;
log() << "autosplitting " << _ns << " size: " << size << " shard: " << toString() << endl;
Chunk * newShard = split();
moveIfShould( newShard );
return true;
}
void DCDataSet::write(Dimensions srcBuffer, Dimensions srcStride,
Dimensions srcOffset, Dimensions srcData,
Dimensions dstOffset, const void* data)
throw (DCException)
{
log_msg(2, "DCDataSet::write (%s)", name.c_str());
if (!opened)
throw DCException(getExceptionString("write: Dataset has not been opened/created"));
log_msg(3,
" ndims = %llu\n"
" logical_size = %s\n"
" physical_size = %s\n"
" src_buffer = %s\n"
" src_stride = %s\n"
" src_data = %s\n"
" src_offset = %s\n"
" dst_offset = %s\n",
(long long unsigned) ndims,
getLogicalSize().toString().c_str(),
getPhysicalSize().toString().c_str(),
srcBuffer.toString().c_str(),
srcStride.toString().c_str(),
srcData.toString().c_str(),
srcOffset.toString().c_str(),
dstOffset.toString().c_str());
// swap dimensions if necessary
srcBuffer.swapDims(ndims);
srcStride.swapDims(ndims);
srcData.swapDims(ndims);
srcOffset.swapDims(ndims);
dstOffset.swapDims(ndims);
// dataspace to read from
hid_t dsp_src;
if (getLogicalSize().getScalarSize() != 0)
{
dsp_src = H5Screate_simple(ndims, srcBuffer.getPointer(), NULL);
if (dsp_src < 0)
throw DCException(getExceptionString("write: Failed to create source dataspace"));
if (H5Sselect_hyperslab(dsp_src, H5S_SELECT_SET, srcOffset.getPointer(),
srcStride.getPointer(), srcData.getPointer(), NULL) < 0 ||
H5Sselect_valid(dsp_src) <= 0)
throw DCException(getExceptionString("write: Invalid source hyperslap selection"));
if (srcData.getScalarSize() == 0)
H5Sselect_none(dsp_src);
// dataspace to write to
if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, dstOffset.getPointer(),
NULL, srcData.getPointer(), NULL) < 0 ||
H5Sselect_valid(dataspace) <= 0)
throw DCException(getExceptionString("write: Invalid target hyperslap selection"));
if (!data || (srcData.getScalarSize() == 0))
{
H5Sselect_none(dataspace);
data = NULL;
}
// write data to the dataset
if (H5Dwrite(dataset, this->datatype, dsp_src, dataspace, dsetWriteProperties, data) < 0)
throw DCException(getExceptionString("write: Failed to write dataset"));
H5Sclose(dsp_src);
}
}
void DCDataSet::read(Dimensions dstBuffer,
Dimensions dstOffset,
Dimensions srcSize,
Dimensions srcOffset,
Dimensions& sizeRead,
uint32_t& srcNDims,
void* dst)
throw (DCException)
{
log_msg(2, "DCDataSet::read (%s)", name.c_str());
if (!opened)
throw DCException(getExceptionString("read: Dataset has not been opened/created"));
if (dstBuffer.getScalarSize() == 0)
dstBuffer.set(srcSize);
// dst buffer is allowed to be NULL
// in this case, only the size of the dataset is returned
// if the dataset is empty, return just its size as there is nothing to read
if ((dst != NULL) && (getNDims() > 0))
{
log_msg(3,
" ndims = %llu\n"
" logical_size = %s\n"
" physical_size = %s\n"
" dstBuffer = %s\n"
" dstOffset = %s\n"
" srcSize = %s\n"
" srcOffset = %s\n",
(long long unsigned) ndims,
getLogicalSize().toString().c_str(),
getPhysicalSize().toString().c_str(),
dstBuffer.toString().c_str(),
dstOffset.toString().c_str(),
srcSize.toString().c_str(),
srcOffset.toString().c_str());
dstBuffer.swapDims(ndims);
dstOffset.swapDims(ndims);
srcSize.swapDims(ndims);
srcOffset.swapDims(ndims);
hid_t dst_dataspace = H5Screate_simple(ndims, dstBuffer.getPointer(), NULL);
if (dst_dataspace < 0)
throw DCException(getExceptionString("read: Failed to create target dataspace"));
if (H5Sselect_hyperslab(dst_dataspace, H5S_SELECT_SET, dstOffset.getPointer(), NULL,
srcSize.getPointer(), NULL) < 0 ||
H5Sselect_valid(dst_dataspace) <= 0)
throw DCException(getExceptionString("read: Target dataspace hyperslab selection is not valid!"));
if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, srcOffset.getPointer(), NULL,
srcSize.getPointer(), NULL) < 0 ||
H5Sselect_valid(dataspace) <= 0)
throw DCException(getExceptionString("read: Source dataspace hyperslab selection is not valid!"));
if (srcSize.getScalarSize() == 0)
H5Sselect_none(dataspace);
if (H5Dread(dataset, this->datatype, dst_dataspace, dataspace, dsetReadProperties, dst) < 0)
throw DCException(getExceptionString("read: Failed to read dataset"));
H5Sclose(dst_dataspace);
srcSize.swapDims(ndims);
}
// swap dimensions if necessary
sizeRead.set(srcSize);
srcNDims = this->ndims;
log_msg(3, " returns ndims = %llu", (long long unsigned) ndims);
log_msg(3, " returns sizeRead = %s", sizeRead.toString().c_str());
}
