std::vector<Variant> PropertyHDF5::values(void) const
{
std::vector<Variant> values;
DataSet dset = dataset();
DataType dtype = data_type_from_h5(dset.dataType());
NDSize shape = dset.size();
if (shape.size() < 1 || shape[0] < 1) {
return values;
}
assert(shape.size() == 1);
size_t nvalues = nix::check::fits_in_size_t(shape[0], "Can't resize: data to big for memory");
switch (dtype) {
case DataType::Bool: do_read_value<bool>(dset, nvalues, values); break;
case DataType::Int32: do_read_value<int32_t>(dset, nvalues, values); break;
case DataType::UInt32: do_read_value<uint32_t>(dset, nvalues, values); break;
case DataType::Int64: do_read_value<int64_t>(dset, nvalues, values); break;
case DataType::UInt64: do_read_value<uint64_t>(dset, nvalues, values); break;
case DataType::String: do_read_value<char *>(dset, nvalues, values); break;
case DataType::Double: do_read_value<double>(dset, nvalues, values); break;
#ifndef CHECK_SUPOORTED_VALUES
default: assert(DATATYPE_SUPPORT_NOT_IMPLEMENTED);
#endif
}
return values;
}
DataSet DataSet::create(const H5::CommonFG &parent,
const std::string &name,
const H5::DataType &fileType,
const NDSize &size,
const NDSize &maxsize,
const NDSize &chunks,
bool max_size_unlimited,
bool guess_chunks)
{
H5::DataSpace space;
if (size) {
if (maxsize) {
space = DataSpace::create(size, maxsize);
} else {
space = DataSpace::create(size, max_size_unlimited);
}
}
H5::DSetCreatPropList plcreate = H5::DSetCreatPropList::DEFAULT;
if (chunks) {
int rank = static_cast<int>(chunks.size());
plcreate.setChunk(rank, chunks.data());
} else if (guess_chunks) {
NDSize guessedChunks = DataSet::guessChunking(size, fileType.getSize());
plcreate.setChunk(static_cast<int>(guessedChunks.size()), guessedChunks.data());
}
H5::DataSet dset = parent.createDataSet(name, fileType, space);
return DataSet(dset);
}
double psize_product(const NDSize &dims)
{
double product = 1;
std::for_each(dims.begin(), dims.end(), [&](hsize_t val) {
product *= val;
});
return product;
}
DataArray createDataArray(const std::string &name,
const std::string &type,
const T &data) {
const Hydra<const T> hydra(data);
DataType dtype = hydra.element_data_type();
const NDSize shape = hydra.shape();
DataArray da = createDataArray(name, type, dtype, shape);
const NDSize offset(shape.size(), 0);
da.setData(data, offset);
return da;
}
void DataArray::ioRead(DataType dtype, void *data, const NDSize &count, const NDSize &offset) const {
const std::vector<double> poly = polynomCoefficients();
boost::optional<double> opt_origin = expansionOrigin();
if (poly.size() || opt_origin) {
size_t data_esize = data_type_to_size(dtype);
size_t nelms = check::fits_in_size_t(count.nelms(),
"Cannot apply polynom or oirign transform. Buffer needed exceeds memory.");
std::vector<double> tmp;
double *read_buffer;
if (data_esize < sizeof(double)) {
//need temporary buffer
tmp.resize(nelms);
read_buffer = tmp.data();
} else {
read_buffer = reinterpret_cast<double *>(data);
}
getDataDirect(DataType::Double, read_buffer, count, offset);
const double origin = opt_origin ? *opt_origin : 0.0;
util::applyPolynomial(poly, origin, read_buffer, read_buffer, nelms);
convertData(DataType::Double, dtype, read_buffer, nelms);
if (tmp.size()) {
memcpy(data, read_buffer, nelms * data_esize);
}
} else {
getDataDirect(dtype, data, count, offset);
}
}
/**
* Infer the chunk size from the supplied size information
*
* @param chunks Size information to base the guessing on
* @param elementSize The size of a single element in bytes
*
* This function is a port of the guess_chunk() function from h5py
* low-level Python interface to the HDF5 library.\n
* http://h5py.alfven.org\n
*
* @copyright Copyright 2008 - 2013 Andrew Collette & contributers\n
* License: BSD 3-clause (see LICENSE.h5py)\n
*
* @return An (maybe not at all optimal) guess for chunk size
*/
NDSize DataSet::guessChunking(NDSize chunks, size_t element_size)
{
// original source:
// https://github.com/h5py/h5py/blob/2.1.3/h5py/_hl/filters.py
if (chunks.size() == 0) {
throw InvalidRank("Cannot guess chunks for 0-dimensional data");
}
double product = 1;
std::for_each(chunks.begin(), chunks.end(), [&](hsize_t &val) {
//todo: check for +infinity
if (val == 0)
val = 1024;
product *= val;
});
product *= element_size;
double target_size = CHUNK_BASE * pow(2, log10(product/(1024.0 * 1024.0)));
if (target_size > CHUNK_MAX)
target_size = CHUNK_MAX;
else if (target_size < CHUNK_MIN)
target_size = CHUNK_MIN;
size_t i = 0;
while (true) {
double csize = static_cast<double>(chunks.nelms());
if (csize == 1.0) {
break;
}
double cbytes = csize * element_size;
if ((cbytes < target_size || (std::abs(cbytes - target_size) / target_size) < 0.5)
&& cbytes < CHUNK_MAX) {
break;
}
//not done yet, one more iteration
size_t idx = i % chunks.size();
if (chunks[idx] > 1) {
chunks[idx] = chunks[idx] >> 1; //divide by two
}
i++;
}
bool positionInData(const DataArray &data, const NDSize &position) {
NDSize data_size = data.dataExtent();
bool valid = true;
if (!(data_size.size() == position.size())) {
return false;
}
for (size_t i = 0; i < data_size.size(); i++) {
valid &= position[i] < data_size[i];
}
return valid;
}
void DataArray::appendData(DataType dtype, const void *data, const NDSize &count, size_t axis) {
//first some sanity checks
NDSize extent = dataExtent();
if (axis >= extent.size()) {
throw InvalidRank("axis is out of bounds");
}
if (extent.size() != count.size()) {
throw IncompatibleDimensions("Data and DataArray must have the same dimensionality", "appendData");
}
for (size_t i = 0; i < count.size(); i ++) {
if (i == axis) {
continue;
}
if (extent[i] != count[i]) {
throw IncompatibleDimensions("Shape of data and shape of DataArray must match in all dimension but axis!",
"appenData");
}
}
NDSize offset(extent.size(), 0);
offset[axis] = extent[axis];
extent[axis] += count[axis];
//enlarge the DataArray to fit the new data
dataExtent(extent);
setData(dtype, data, count, offset);
}
void TestDataSet::testNDSize() {
NDSize invalidSize = {};
NDSize a({23, 42, 1982});
CPPUNIT_ASSERT(!invalidSize); // testing operator bool()
CPPUNIT_ASSERT(a ? true : false);
typedef NDSize::value_type value_type;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(23), a[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(42), a[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1982), a[2]);
CPPUNIT_ASSERT_THROW(a[3], std::out_of_range);
a++;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(24), a[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(43), a[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1983), a[2]);
a--;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(23), a[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(42), a[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1982), a[2]);
a += 13;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(36), a[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(55), a[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1995), a[2]);
a -= 13;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(23), a[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(42), a[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1982), a[2]);
NDSize b({19, 1940, 18});
NDSize c = a + b;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(42), c[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1982), c[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(2000), c[2]);
NDSize d = c - b;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(23), d[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(42), d[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1982), d[2]);
NDSize f({1, 2, 3, 4});
CPPUNIT_ASSERT_THROW(a + f, std::out_of_range);
NDSize g(f.size(), 0);
g += f;
CPPUNIT_ASSERT(g == f);
CPPUNIT_ASSERT(g != a);
NDSize h = b / b;
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1), h[0]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1), h[1]);
CPPUNIT_ASSERT_EQUAL(static_cast<value_type>(1), h[2]);
NDSize j(h.size(), static_cast<value_type>(333));
NDSize k = h * j;
CPPUNIT_ASSERT(j == k);
NDSize::value_type dp = j.dot(h);
CPPUNIT_ASSERT_EQUAL(static_cast<NDSize::value_type>(999), dp);
NDSize s({3, 4});
dp = s.dot(s);
CPPUNIT_ASSERT_EQUAL(static_cast<NDSize::value_type>(25), dp);
//comparison tests
CPPUNIT_ASSERT_THROW(f < s, IncompatibleDimensions);
NDSize t({4, 5});
NDSize u({4, 4});
// actual non-delegation implementations are < and <=
CPPUNIT_ASSERT(s < t);
CPPUNIT_ASSERT(u <= t);
// everything else, i.e. >, >= is delegated
CPPUNIT_ASSERT(t > s);
CPPUNIT_ASSERT(t >= u);
CPPUNIT_ASSERT(!(t <= s));
CPPUNIT_ASSERT(!(t < u));
}
