void Bundle2::storeParameters(H5::H5File& file) const {
H5::Group root = file.openGroup("/");
H5::DataSpace scalar;
H5::Attribute attr = root.createAttribute("version", H5::PredType::STD_U32LE, scalar);
attr.write(H5::PredType::NATIVE_UINT, &version_);
attr.close();
unsigned char r2 = parameters_.reduce2?1:0;
attr = root.createAttribute("reduce2", H5::PredType::STD_U8LE, scalar);
attr.write(H5::PredType::NATIVE_UCHAR, &r2);
attr.close();
attr = root.createAttribute("xROI", H5::PredType::STD_U32LE, scalar);
attr.write(H5::PredType::NATIVE_UINT, ¶meters_.xROI);
attr.close();
attr = root.createAttribute("yROI", H5::PredType::STD_U32LE, scalar);
attr.write(H5::PredType::NATIVE_UINT, ¶meters_.yROI);
attr.close();
scalar.close();
root.close();
}
void
DataSetIO::SaveDataSet(const DataSet& dataSet_, const std::string& filename_){
// Get the relevant params
hsize_t nEntries = dataSet_.GetNEntries();
hsize_t nObs = dataSet_.GetNObservables();
hsize_t nData = nObs * nEntries;
// create colon separated string from list of observables
std::vector<std::string> observableNames = dataSet_.GetObservableNames();
if (observableNames.size() != nObs)
throw HdfIOError("SaveDataSet::Require one name and one name only for each observable");
// Set up files
H5::H5File file(filename_, H5F_ACC_TRUNC);
// Flatten data into 1D array
// HDF5 likes c arrays. Here use a vector and pass pointer to first element
// memory guaranteed to be contiguous
std::vector<double> flattenedData;
std::vector<double> eventData;
flattenedData.reserve(nData);
for(size_t i = 0; i < nEntries; i++){
eventData = dataSet_.GetEntry(i).GetData();
flattenedData.insert(flattenedData.end(), eventData.begin(), eventData.end());
}
// Set up the data set
// 1D, ndata long, called "observations". Saved as native doubles on this computer
H5::DataSpace dataSpace(1, &nData);
H5::DataSet theData(file.createDataSet("observations", H5::PredType::NATIVE_DOUBLE, dataSpace));
// Set up the attributes - the number of obs per event and the names of the observables
// 64 chars max in str to save space
H5::StrType strType(H5::PredType::C_S1, 64);
H5::DataSpace attSpace(H5S_SCALAR);
H5::Attribute obsListAtt = theData.createAttribute("observed_quantities", strType, attSpace);
obsListAtt.write(strType, FlattenStringVector(observableNames, fDelimiter));
H5::Attribute countAtt = theData.createAttribute("n_observables",
H5::PredType::NATIVE_INT,
attSpace);
countAtt.write(H5::PredType::NATIVE_INT, &nObs);
// Write the data
theData.write(&flattenedData.at(0), H5::PredType::NATIVE_DOUBLE);
}
void save_string_attr(H5::Group &g, const char *name, const char *val)
{
H5::DataSpace string_space(H5S_SCALAR);
H5::StrType strdatatype(H5::PredType::C_S1, strlen(val)+1);
H5::Attribute attr = g.createAttribute(name, strdatatype, string_space);
attr.write(strdatatype, val);
}
inline typename boost::enable_if<boost::mpl::and_<is_array<T>, boost::is_same<typename T::value_type, char const*> >, void>::type
write_attribute(H5::H5Object const& object, std::string const& name, T const& value)
{
enum { size = T::static_size };
hsize_t dim[1] = { size };
H5::DataSpace ds(1, dim);
size_t max_len = 0;
for (size_t i = 0; i < size; ++i) {
max_len = std::max(max_len, strlen(value[i]));
}
H5::StrType tid(H5::PredType::C_S1, max_len);
// remove attribute if it exists
try {
H5XX_NO_AUTO_PRINT(H5::AttributeIException);
object.removeAttr(name);
}
catch (H5::AttributeIException const&) {}
H5::Attribute attr = object.createAttribute(name, tid, ds);
std::vector<char> data(max_len * size);
for (size_t i = 0; i < size; ++i) {
strncpy(&*data.begin() + i * max_len, value[i], max_len);
}
attr.write(tid, &*data.begin());
}
inline typename boost::enable_if<boost::mpl::and_<
is_vector<T>
, boost::is_same<typename T::value_type, std::string>
>, void>::type
write_attribute(H5::H5Object const& object, std::string const& name, T const& value)
{
size_t size = value.size();
// size of longest string
size_t str_len = 0;
for (size_t i = 0; i < size; ++i) {
str_len = std::max(str_len, value[i].size());
}
// remove attribute if it exists and re-create with proper String datatype
// and simple dataspace
if (exists_attribute(object, name)) {
object.removeAttr(name);
}
hsize_t dim[1] = { size };
H5::DataSpace ds(1, dim);
H5::StrType tid(H5::PredType::C_S1, str_len);
H5::Attribute attr = object.createAttribute(name, tid, ds);
// copy strings to contiguous buffer
std::vector<char> buffer(size * str_len);
for (size_t i = 0; i < size; ++i) {
value[i].copy(buffer.data() + i * str_len, str_len);
}
attr.write(tid, &*buffer.begin());
}
void Create(H5::H5Location &object, const std::string & name, const std::vector<std::string> &vect) {
hsize_t length = vect.size();
H5::StrType strType(0,H5T_VARIABLE);
H5::ArrayType arrayDataType(strType, 1, &length);
attribute = object.createAttribute(name.c_str(), strType, H5::DataSpace(1, &length));
attribute.write(strType, &((vect)[0]));
}
void Datafile::add_attribute(const char* name, int value) {
try {
H5::Attribute attr = root_group.createAttribute(name, int_type, scalar_space);
attr.write(int_type, &value);
}
catch (H5::Exception& e) {
e.printError();
throw;
}
}
void Datafile::add_attribute(const char* name, unsigned long int value) {
try {
H5::Attribute attr = root_group.createAttribute(name, H5::PredType::NATIVE_ULONG, scalar_space);
attr.write(H5::PredType::NATIVE_ULONG, &value);
}
catch (H5::Exception& e) {
e.printError();
throw;
}
}
void WriteTAttrib(const std::string& name, const T& value,
const H5::DataType& dType, const H5::DataSpace& dSpace){
//attributes are clunky in HDF5++ implementation - this is a workaround
//template is required to pass the proper value type
//create a new data set attribute
H5::Attribute attrib = dSet_.createAttribute(name,dType,dSpace);
//write the value to the attribute and close
attrib.write(dType,&value);
attrib.close();
}
void Attribute::write(H5::H5File f, const cpath & dataset_root)
{
//FIXME we should have a path?
cpath fullpath = dataset_root / name;
cpath grouppath = fullpath.parent_path();
if (_link.size())
{
if (!h5_obj_exists(f, grouppath))
h5_create_path_groups(f, grouppath.c_str());
H5::Group g = f.openGroup(grouppath.generic_string().c_str());
if (h5_obj_exists(f, fullpath))
g.unlink(name.filename().generic_string().c_str());
g.link(H5G_LINK_SOFT, (dataset_root/_link).generic_string().c_str(), name.filename().generic_string().c_str());
}
else if (_m.total() == 0) {
//FIXME remove this (legacy) case
hsize_t *dim = new hsize_t[size.size()+1];
for(uint i=0;i<size.size();i++)
dim[i] = size[i];
H5::DataSpace space(size.size(), dim);
H5::Attribute attr;
H5::Group g;
delete[] dim;
if (!h5_obj_exists(f, grouppath))
h5_create_path_groups(f, grouppath);
g = f.openGroup(grouppath.generic_string().c_str());
uint min, max;
H5Pget_attr_phase_change(H5Gget_create_plist(g.getId()), &max, &min);
if (min || max)
printf("WARNING: could not set dense storage on group, may not be able to write large attributes\n");
//FIXME relative to what?
if (H5Aexists(g.getId(), name.filename().generic_string().c_str()))
g.removeAttr(name.filename().generic_string().c_str());
attr = g.createAttribute(name.filename().generic_string().c_str(), toH5DataType(type), space);
attr.write(toH5NativeDataType(type), data);
}
else
Mat_H5AttrWrite(_m, f, fullpath);
}
inline typename boost::enable_if<boost::is_same<T, char const*>, void>::type
write_attribute(H5::H5Object const& object, std::string const& name, T value)
{
H5::StrType tid(H5::PredType::C_S1, strlen(value));
// remove attribute if it exists
try {
H5XX_NO_AUTO_PRINT(H5::AttributeIException);
object.removeAttr(name);
}
catch (H5::AttributeIException const&) {}
H5::Attribute attr = object.createAttribute(name, tid, H5S_SCALAR);
attr.write(tid, value);
}
void CompartmentReportHDF5::_createDataAttributes( H5::DataSet& dataset )
{
const int rank = 0;
const double startTime = getStartTime();
const double endTime = getEndTime();
const double timestep = getTimestep();
H5::Attribute rankAttr = dataset.createAttribute( dataAttributes[0],
H5::PredType::NATIVE_INT, H5S_SCALAR );
H5::Attribute tstartAttr = dataset.createAttribute( dataAttributes[1],
H5::PredType::NATIVE_DOUBLE, H5S_SCALAR );
H5::Attribute tstopAttr = dataset.createAttribute( dataAttributes[2],
H5::PredType::NATIVE_DOUBLE, H5S_SCALAR );
H5::Attribute dtAttr = dataset.createAttribute( dataAttributes[3],
H5::PredType::NATIVE_DOUBLE, H5S_SCALAR );
rankAttr.write( H5::PredType::NATIVE_INT, &rank );
tstartAttr.write( H5::PredType::NATIVE_DOUBLE, &startTime );
tstopAttr.write( H5::PredType::NATIVE_DOUBLE, &endTime );
dtAttr.write( H5::PredType::NATIVE_DOUBLE, ×tep );
detail::addStringAttribute( dataset, dataAttributes[4], _dunit );
detail::addStringAttribute( dataset, dataAttributes[5], _tunit );
}
void WriteAttrib(const std::string& name, const T& value,
const H5::DataType& dType, const H5::DataSpace& dSpace){
//attributes are clunky in HDF5++ implementation - this is a workaround
//template is required to pass the proper value type
//access the built-in root group and create a new attribute for it.
H5::Group rootGroup = file_->openGroup("/");
H5::Attribute attrib = rootGroup.createAttribute(name,dType,dSpace);
//write the value to the attribute and close
attrib.write(dType,&value);
attrib.close();
}
void Datafile::add_attribute(const char* name, std::string const& value) {
const size_t len = value.length();
try {
H5::AtomType string_type = H5::AtomType(H5::PredType::C_S1);
// setSize does not like zero sizes
if (len != 0)
string_type.setSize(len);
H5::Attribute attr = root_group.createAttribute(name, string_type, scalar_space);
attr.write(string_type, value.c_str());
}
catch (H5::Exception& e) {
e.printError();
throw;
}
}
void WriteTStrAttrib(const std::string& name, const std::string& value){
std::string str = value;
str.resize( STRING_ATTRIB_SIZE );
//define the data type as a string with the value's length
H5::StrType strType(0, STRING_ATTRIB_SIZE );
//open the data set and create a new attribute of type string
H5::Attribute attrib =
dSet_.createAttribute(name, strType, H5::DataSpace(H5S_SCALAR));
//write value to the attribute and close
attrib.write(strType, str );
attrib.close();
}
void Bundle2::closeSaveStream() {
H5::DataSpace scalar;
// Saving remaining POI information
H5::Group poiGroup = streamFile_->openGroup("/POI");
H5::Attribute attr = poiGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
hsize_t count = poiFirstFrame_;
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
poiGroup.close();
scalar.close();
// Closing HDF5 file
streamFile_->close();
delete streamFile_;
streamFile_ = NULL;
}
inline typename boost::enable_if<boost::is_fundamental<T>, void>::type
write_attribute(H5::H5Object const& object, std::string const& name, T const& value)
{
H5::Attribute attr;
try {
H5XX_NO_AUTO_PRINT(H5::AttributeIException);
attr = object.openAttribute(name);
if (!has_type<T>(attr) || !has_scalar_space(attr)) {
// recreate attribute with proper type
object.removeAttr(name);
throw H5::AttributeIException();
}
}
catch (H5::AttributeIException const&) {
attr = object.createAttribute(name, ctype<T>::hid(), H5S_SCALAR);
}
attr.write(ctype<T>::hid(), &value);
}
void WriteStrAttrib(const std::string& name, const std::string& value){
//HDF5 has a mangled way of creating string attributes -
//this is a workaround
//create a string array of length 1 containing the value to be written
//std::string strBuf[1] = {value};
std::string str = value;
str.resize( STRING_ATTRIB_SIZE );
//define the data type as a string with the value's length
H5::StrType strType(0, STRING_ATTRIB_SIZE );
//open the root group and create a new attribute of type string
H5::Group rootGroup = file_->openGroup("/");
H5::Attribute attrib = rootGroup.createAttribute(name,
strType,H5::DataSpace());
//write value to the attribute and close
attrib.write(strType, str);
attrib.close();
}
inline typename boost::enable_if<boost::mpl::and_<is_array<T>, boost::is_fundamental<typename T::value_type> >, void>::type
write_attribute(H5::H5Object const& object, std::string const& name, T const& value)
{
typedef typename T::value_type value_type;
enum { size = T::static_size };
H5::Attribute attr;
try {
H5XX_NO_AUTO_PRINT(H5::AttributeIException);
attr = object.openAttribute(name);
if (!has_type<T>(attr) || !has_extent<T>(attr)) {
// recreate attribute with proper type and size
object.removeAttr(name);
throw H5::AttributeIException();
}
}
catch (H5::AttributeIException const&) {
hsize_t dim[1] = { size };
H5::DataSpace ds(1, dim);
attr = object.createAttribute(name, ctype<value_type>::hid(), ds);
}
attr.write(ctype<value_type>::hid(), &*value.begin());
}
void Bundle2::streamPOI(size_t frame) {
H5::DataSpace scalar;
H5::Group poiGroup = streamFile_->openGroup("/POI");
for(size_t i = poiFirstFrame_; i <= frame; ++i) {
const std::string frameGroupName = boost::str(boost::format("Frame %1$04d") % i);
H5::Group frameGroup = poiGroup.createGroup(frameGroupName);
hsize_t count = poi_[(ptrdiff_t)i - (ptrdiff_t)poiFirstFrame_].size();
H5::Attribute attr = frameGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
for(size_t camera = 0; camera < poi_[(ptrdiff_t)i - (ptrdiff_t)poiFirstFrame_].size(); ++camera)
poi_[(ptrdiff_t)i - (ptrdiff_t)poiFirstFrame_][camera].save(frameGroup, camera);
frameGroup.close();
}
poiGroup.close();
scalar.close();
poi_.erase(poi_.begin(), poi_.begin() + (ptrdiff_t)frame - (ptrdiff_t)poiFirstFrame_ + 1);
poiFirstFrame_ = frame + 1;
}
inline typename boost::enable_if<is_multi_array<T>, void>::type
write_attribute(H5::H5Object const& object, std::string const& name, T const& value)
{
typedef typename T::element value_type;
enum { rank = T::dimensionality };
H5::Attribute attr;
try {
H5XX_NO_AUTO_PRINT(H5::AttributeIException);
attr = object.openAttribute(name);
if (!has_type<T>(attr) || !has_extent<T>(attr, value.shape())) {
// recreate attribute with proper type and size
object.removeAttr(name);
throw H5::AttributeIException();
}
}
catch (H5::AttributeIException const&) {
hsize_t dim[rank];
std::copy(value.shape(), value.shape() + rank, dim);
H5::DataSpace ds(rank, dim);
attr = object.createAttribute(name, ctype<value_type>::hid(), ds);
}
attr.write(ctype<value_type>::hid(), value.origin());
}
output_t :: output_t(config_t config_) : sn_config(config_),
file(sn_config.get_outfile_name().data(), H5F_ACC_TRUNC),
dset_dims{sn_config.get_num_xi(), sn_config.get_nelem()},
file_space(2, dset_dims)
{
// We do chunked access to the dataset, see
// https://www.hdfgroup.org/HDF5/doc/Advanced/Chunking/index.html
// https://www.hdfgroup.org/HDF5/doc/cpplus_RM/writedata_8cpp-example.html
// Create a dataset creation property list, use chunking
constexpr double fill_val{-1.0};
prop_list.setFillValue(H5::PredType::NATIVE_DOUBLE, &fill_val);
// Create dataset and write to file
dataset = new H5::DataSet(file.createDataSet("shotnoise", H5::PredType::NATIVE_DOUBLE, file_space, prop_list));
// Write the configuration file as attributes
map<dist_t, std::string> my_map{
{dist_t::expon_t, string("exponential")},
{dist_t::normal_t, string("normal")},
{dist_t::uniform_t, string("uniform")}};
int att_val_int{0}; // Store attribute value for integer values
double att_val_double{0.0}; // Store attribute value for double values
vector<double> att_vec_double;
const hsize_t att_vec_dims{2};
H5std_string att_str; // Store attribute value for string values
// Create new string datatype for attribute
H5::StrType strdatatype(H5::PredType::C_S1, 32); // of length 32 characters
H5::DataSpace attrib_space(H5S_SCALAR); // Dataspace for scalar attributes
H5::DataSpace attrib_space2(1, &att_vec_dims); // Dataspace for 2 element vectors
H5::Attribute att = dataset -> createAttribute("num_bursts", H5::PredType::NATIVE_INT, attrib_space);
att_val_int = sn_config.get_num_bursts();
att.write(H5::PredType::NATIVE_INT, &att_val_int);
att = dataset -> createAttribute("dt", H5::PredType::NATIVE_DOUBLE, attrib_space);
att_val_double = sn_config.get_dt();
att.write(H5::PredType::NATIVE_DOUBLE, &att_val_double);
att = dataset -> createAttribute("Tend", H5::PredType::NATIVE_DOUBLE, attrib_space);
att_val_double = sn_config.get_Tend();
att.write(H5::PredType::NATIVE_DOUBLE, &att_val_double);
att = dataset -> createAttribute("Lpar", H5::PredType::NATIVE_DOUBLE, attrib_space);
att_val_double = sn_config.get_Lpar();
att.write(H5::PredType::NATIVE_DOUBLE, &att_val_double);
att = dataset -> createAttribute("Cs", H5::PredType::NATIVE_DOUBLE, attrib_space);
att_val_double = sn_config.get_Cs();
att.write(H5::PredType::NATIVE_DOUBLE, &att_val_double);
att = dataset -> createAttribute("Lsol", H5::PredType::NATIVE_DOUBLE, attrib_space);
att_val_double = sn_config.get_Lsol();
att.write(H5::PredType::NATIVE_DOUBLE, &att_val_double);
att = dataset -> createAttribute("Amplitude distribution", strdatatype, attrib_space);
att_str = my_map.at(sn_config.get_dist_amp_type());
att.write(strdatatype, att_str);
att_vec_double = sn_config.get_dist_amp_params();
att = dataset -> createAttribute("Amplitude distribution parameters", H5::PredType::NATIVE_DOUBLE, attrib_space2);
att.write(H5::PredType::NATIVE_DOUBLE, att_vec_double.data());
att = dataset -> createAttribute("Arrival time distribution", strdatatype, attrib_space);
att_str = my_map.at(sn_config.get_dist_arrival_type());
att.write(strdatatype, att_str);
att_vec_double = sn_config.get_dist_arrival_params();
att = dataset -> createAttribute("Arrival time distribution parameters", H5::PredType::NATIVE_DOUBLE, attrib_space2);
att.write(H5::PredType::NATIVE_DOUBLE, att_vec_double.data());
att = dataset -> createAttribute("Length distribution", strdatatype, attrib_space);
att_str = my_map.at(sn_config.get_dist_length_type());
att.write(strdatatype, att_str);
att_vec_double = sn_config.get_dist_length_params();
att = dataset -> createAttribute("Length distribution parameters", H5::PredType::NATIVE_DOUBLE, attrib_space2);
att.write(H5::PredType::NATIVE_DOUBLE, att_vec_double.data());
att = dataset -> createAttribute("Velocity distribution", strdatatype, attrib_space);
att_str = my_map.at(sn_config.get_dist_vrad_type());
att.write(strdatatype, att_str);
att_vec_double = sn_config.get_dist_vrad_params();
att = dataset -> createAttribute("Velocity distribution parameters", H5::PredType::NATIVE_DOUBLE, attrib_space2);
att.write(H5::PredType::NATIVE_DOUBLE, att_vec_double.data());
}
void Bundle2::initFrameStream_() {
H5::DataSpace scalar;
// Creating datasets for each frame
H5::Group bundleGroup = streamFile_->createGroup("/Bundle");
H5::Group framesGroup = bundleGroup.createGroup("Frames");
hsize_t count = frames_.size();
H5::Attribute attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
// Defining frame dataset property
hsize_t chunkDim[] = { 1, 2, 10 };
H5::DSetCreatPropList propList;
propList.setLayout(H5D_CHUNKED);
propList.setChunk(3, chunkDim);
propList.setDeflate(9);
// Definig dataset dataspace
hsize_t max_dim[] = { numCameras_, 2, H5S_UNLIMITED };
hsize_t dim[] = { numCameras_, 2, 0 };
H5::DataSpace ds(3, dim, max_dim);
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
const std::string datasetName = boost::str(boost::format("Frame %1$04d") % (*it)->number());
// Creating dataset
H5::DataSet frameData = framesGroup.createDataSet(datasetName, H5::PredType::IEEE_F32LE, ds, propList);
// Writing global number
attr = frameData.createAttribute("globalNumber", H5::PredType::STD_U64LE, scalar);
count = (*it)->globalNumber();
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
// Clean up!
frameData.close();
}
// Clean up!
ds.close();
propList.close();
// Creating tracks dataset
hsize_t chunkDim2[] = { 2, 10 };
propList = H5::DSetCreatPropList();
propList.setLayout(H5D_CHUNKED);
propList.setChunk(2, chunkDim);
propList.setDeflate(9);
H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE);
hsize_t tracksDim[] = { 0, 2 };
hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 };
H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim);
H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList);
tracksDataset.close();
tracksDataspace.close();
tracksDatasetType.close();
propList.close();
framesGroup.close();
bundleGroup.close();
scalar.close();
}
void TChainWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& chain, const std::string& meta) {
H5::H5File* h5file = H5Utils::openFile(fname);
H5::Group* h5group = H5Utils::openGroup(h5file, group);
// Dataset properties: optimized for reading/writing entire buffer at once
int rank = 3;
hsize_t dim[3] = {length_, nSamples_+2, nDim_};
H5::DataSpace dspace(rank, &(dim[0]));
H5::DSetCreatPropList plist;
plist.setDeflate(9); // gzip compression level
plist.setChunk(rank, &(dim[0]));
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
H5::DataSet* dataset = NULL;
try {
dataset = new H5::DataSet(h5group->createDataSet(chain, H5::PredType::NATIVE_FLOAT, dspace, plist));
} catch(H5::GroupIException &group_exception) {
std::cerr << "Could not create dataset for chain." << std::endl;
std::cerr << "Dataset '" << group << "/" << chain << "' most likely already exists." << std::endl;
throw;
}
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
if(meta == "") { // Store metadata as attributes
bool *converged = new bool[length_];
float *lnZ = new float[length_];
for(unsigned int i=0; i<length_; i++) {
converged[i] = metadata[i].converged;
lnZ[i] = metadata[i].lnZ;
}
// Allow large attributes to be stored in dense storage, versus compact (which has 64 kB limit)
//if(length_ > 5) {
// hid_t dID = dataset->getCreatePlist().getId();
// herr_t res = H5Pset_attr_phase_change(dID, 0, 0);
// std::cerr << res << std::endl;
// if(res < 0) {
// std::cerr << "Failed to specify dense storage." << std::endl;
// }
//}
H5::DataSpace convSpace(1, &(dim[0]));
H5::Attribute convAtt = dataset->createAttribute("converged", H5::PredType::NATIVE_CHAR, convSpace);
convAtt.write(H5::PredType::NATIVE_CHAR, reinterpret_cast<char*>(converged));
H5::DataSpace lnZSpace(1, &(dim[0]));
H5::Attribute lnZAtt = dataset->createAttribute("ln(Z)", H5::PredType::NATIVE_FLOAT, lnZSpace);
lnZAtt.write(H5::PredType::NATIVE_FLOAT, lnZ);
delete[] converged;
delete[] lnZ;
} else { // Store metadata as separate dataset
H5::CompType metaType(sizeof(TChainMetadata));
metaType.insertMember("converged", HOFFSET(TChainMetadata, converged), H5::PredType::NATIVE_CHAR);
metaType.insertMember("ln(Z)", HOFFSET(TChainMetadata, lnZ), H5::PredType::NATIVE_FLOAT);
rank = 1;
H5::DataSpace metaSpace(rank, &(dim[0]));
H5::DSetCreatPropList metaProp;
TChainMetadata emptyMetadata = {0, 0};
metaProp.setFillValue(metaType, &emptyMetadata);
metaProp.setDeflate(9);
metaProp.setChunk(rank, &(dim[0]));
H5::DataSet* metaDataset = new H5::DataSet(h5group->createDataSet(meta, metaType, metaSpace, metaProp));
metaDataset->write(metadata.data(), metaType);
delete metaDataset;
metaDataset = NULL;
}
delete dataset;
delete h5group;
delete h5file;
//std::cerr << "Cleaned up." << std::endl;
}
// Bundle management
void Bundle2::save(const boost::filesystem::path& fileName) const {
// Creating HDF5 file
H5::H5File bundleFile(fileName.string(), H5F_ACC_TRUNC);
storeParameters(bundleFile);
H5::DataSpace scalar;
// Saving POI
H5::Group poiGroup = bundleFile.createGroup("/POI");
H5::Attribute attr = poiGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
hsize_t count = poi_.size();
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
for(size_t frame = 0; frame < poi_.size(); ++frame) {
const std::string frameGroupName = boost::str(boost::format("Frame %1$04d") % frame);
H5::Group frameGroup = poiGroup.createGroup(frameGroupName);
count = poi_[frame].size();
attr = frameGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
for(size_t camera = 0; camera < poi_[frame].size(); ++camera)
poi_[frame][camera].save(frameGroup, camera);
frameGroup.close();
}
poiGroup.close();
// Saving key frames
H5::Group bundleGroup = bundleFile.createGroup("/Bundle");
H5::Group framesGroup = bundleGroup.createGroup("Frames");
count = frames_.size();
attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
(*it)->save(framesGroup);
}
framesGroup.close();
// Saving tracks
const hsize_t chunkDim[] = { 2, 1 };
H5::DSetCreatPropList propList;
propList.setLayout(H5D_CHUNKED);
propList.setChunk(2, chunkDim);
propList.setDeflate(9);
H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE);
hsize_t tracksDim[] = { tracks_.size(), 2 };
hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 };
H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim);
H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList);
for(size_t i = 0; i < tracks_.size(); ++i)
tracks_[i]->save(tracksDataset, i);
tracksDataset.close();
tracksDataspace.close();
tracksDatasetType.close();
propList.close();
bundleGroup.close();
scalar.close();
bundleFile.close();
}
bool saveStackHDF5( const char* fileName, const My4DImage& img, Codec_Mapping* mapping )
{
try
{
#ifdef USE_HDF5
H5::Exception::dontPrint();
H5::H5File file( fileName, H5F_ACC_TRUNC );
H5::Group* group = new H5::Group( file.createGroup( "/Channels" ) );
Image4DProxy<My4DImage> proxy( const_cast<My4DImage*>( &img ) );
long scaledHeight = nearestPowerOfEight( proxy.sy );
long scaledWidth = nearestPowerOfEight( proxy.sx );
// Initialize the upper and lower bounds
long pad_right = ( scaledWidth - proxy.sx ) ;
long pad_bottom = ( scaledHeight - proxy.sy );
hsize_t dims[1] = { 1 };
H5::DataSpace attr_ds = H5::DataSpace( 1, dims );
H5::Attribute attr = group->createAttribute( "width", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( proxy.sx ) );
attr = group->createAttribute( "height", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( proxy.sy ) );
attr = group->createAttribute( "frames", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( proxy.sz ) );
attr = group->createAttribute( "pad_right", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( pad_right ) );
attr = group->createAttribute( "pad_bottom", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( pad_bottom ) );
Codec_Mapping* imap = mapping;
if ( !mapping )
{
imap = new Codec_Mapping();
generate_codec_mapping( *imap, proxy.sc );
}
for ( int c = 0; c < proxy.sc; ++c )
{
double default_irange = 1.0; // assumes data range is 0-255.0
if ( proxy.su > 1 )
{
default_irange = 1.0 / 16.0; // 0-4096, like our microscope images
}
std::vector<double> imin( proxy.sc, 0.0 );
std::vector<double> irange2( proxy.sc, default_irange );
// rescale if converting from 16 bit to 8 bit
if ( proxy.su > 1 )
{
if ( img.p_vmin && img.p_vmax )
proxy.set_minmax( img.p_vmin, img.p_vmax );
if ( proxy.has_minmax() )
{
imin[c] = proxy.vmin[c];
irange2[c] = 255.0 / ( proxy.vmax[c] - proxy.vmin[c] );
}
}
FFMpegEncoder encoder( NULL, scaledWidth, scaledHeight,
( *imap )[c].first, ( *imap )[c].second );
// If the image needs padding, fill the expanded border regions with black
for ( int z = 0; z < proxy.sz; ++z )
{
for ( int y = 0; y < scaledHeight; ++y )
{
for ( int x = 0; x < scaledWidth; ++x )
{
// If inside the area with valid data
if ( x < proxy.sx && y < proxy.sy )
{
int ic = c;
double val = proxy.value_at( x, y, z, ic );
val = ( val - imin[ic] ) * irange2[ic]; // rescale to range 0-255
for ( int cc = 0; cc < 3; ++cc )
encoder.setPixelIntensity( x, y, cc, ( int )val );
}
else
for ( int cc = 0; cc < 3; ++cc )
encoder.setPixelIntensity( x, y, cc, 0 );
}
}
encoder.write_frame();
}
for ( int rem = encoder.encoded_frames(); rem < proxy.sz; rem++ )
encoder.encode();
encoder.close();
hsize_t dims[1];
dims[0] = encoder.buffer_size();
H5::DataSpace dataspace( 1, dims );
std: stringstream name;
name << "Channel_" << c;
H5::DataSet dataset = group->createDataSet( name.str(), H5::PredType::NATIVE_UINT8, dataspace );
dataset.write( encoder.buffer(), H5::PredType::NATIVE_UINT8 );
dataset.close();
std::cout << "Encoded channel is " << encoder.buffer_size() << " bytes." << std::endl;
// Uncomment this if you want to dump the individual movies to a temp file
}
#endif
if ( !mapping )
delete imap;
file.close();
return true;
}
catch ( ... ) {}
return false;
}
void DataSet::writeAttr(const H5::Attribute &attr, H5::DataType mem_type, const NDSize &size, const void *data) {
attr.write(mem_type, data);
}
void Create(H5::H5Location &object, const std::string & name, std::vector<int> &vect) {
hsize_t length = vect.size();
H5::ArrayType arrayDataType(H5::PredType::NATIVE_INT, 1, &length);
attribute = object.createAttribute(name.c_str(), H5::PredType::NATIVE_INT, H5::DataSpace(1, &length));
attribute.write(H5::PredType::NATIVE_INT, &((vect)[0]));
}
void Create(H5::H5Location &object, const std::string & name, const std::string & value) {
H5::StrType strType(0, value.size());
attribute = object.createAttribute(name.c_str(), strType, H5::DataSpace(0,NULL));
isInitialized = true;
attribute.write(strType, value.c_str());
}
void DataSet::writeAttr(const H5::Attribute &attr, H5::DataType mem_type, const NDSize &size, const std::string *data) {
StringReader reader(size, data);
attr.write(mem_type, *reader);
}
