void TImgWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& img) {
H5::H5File* h5file = H5Utils::openFile(fname);
H5::Group* h5group = H5Utils::openGroup(h5file, group);
// Dataset properties: optimized for reading/writing entire buffer at once
int rank = 3;
hsize_t dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
hsize_t chunk_dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
if(length_ > 1000) {
float div = ceil((float)length_ / 1000.);
chunk_dim[0] = (int)ceil(length_ / div);
std::cerr << "! Changing chunk length to " << chunk_dim[0] << " stars." << std::endl;
}
H5::DataSpace dspace(rank, &(dim[0]));
H5::DSetCreatPropList plist;
plist.setDeflate(9); // gzip compression level
plist.setChunk(rank, &(chunk_dim[0]));
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
H5::DataSet* dataset = new H5::DataSet(h5group->createDataSet(img, H5::PredType::NATIVE_FLOAT, dspace, plist));
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
/*
* Attributes
*/
hsize_t att_dim = 2;
H5::DataSpace att_dspace(1, &att_dim);
H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
H5::Attribute att_N = dataset->createAttribute("nPix", att_dtype, att_dspace);
att_N.write(att_dtype, &(rect_.N_bins));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
att_min.write(att_dtype, &(rect_.min));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
att_max.write(att_dtype, &(rect_.max));
delete dataset;
delete h5group;
delete h5file;
}
void CompartmentReportHDF5::_createMappingAttributes( H5::DataSet& dataset )
{
// const std::string type =
// boost::lexical_cast< std::string >( _spec.type( ));
const std::string type = "1"; // COMPARTMENT_REPORT
detail::addStringAttribute( dataset, mappingAttributes[0], type );
dataset.createAttribute( mappingAttributes[1], H5::PredType::NATIVE_INT,
H5S_SCALAR );
dataset.createAttribute( mappingAttributes[2], H5::PredType::NATIVE_INT,
H5S_SCALAR );
dataset.createAttribute( mappingAttributes[3], H5::PredType::NATIVE_INT,
H5S_SCALAR );
dataset.createAttribute( mappingAttributes[4], H5::PredType::NATIVE_INT,
H5S_SCALAR );
dataset.createAttribute( mappingAttributes[5], H5::PredType::NATIVE_INT,
H5S_SCALAR );
}
void WriteTAttrib(const std::string& name, const T& value,
const H5::DataType& dType, const H5::DataSpace& dSpace){
//attributes are clunky in HDF5++ implementation - this is a workaround
//template is required to pass the proper value type
//create a new data set attribute
H5::Attribute attrib = dSet_.createAttribute(name,dType,dSpace);
//write the value to the attribute and close
attrib.write(dType,&value);
attrib.close();
}
void CompartmentReportHDF5::_createDataAttributes( H5::DataSet& dataset )
{
const int rank = 0;
const double startTime = getStartTime();
const double endTime = getEndTime();
const double timestep = getTimestep();
H5::Attribute rankAttr = dataset.createAttribute( dataAttributes[0],
H5::PredType::NATIVE_INT, H5S_SCALAR );
H5::Attribute tstartAttr = dataset.createAttribute( dataAttributes[1],
H5::PredType::NATIVE_DOUBLE, H5S_SCALAR );
H5::Attribute tstopAttr = dataset.createAttribute( dataAttributes[2],
H5::PredType::NATIVE_DOUBLE, H5S_SCALAR );
H5::Attribute dtAttr = dataset.createAttribute( dataAttributes[3],
H5::PredType::NATIVE_DOUBLE, H5S_SCALAR );
rankAttr.write( H5::PredType::NATIVE_INT, &rank );
tstartAttr.write( H5::PredType::NATIVE_DOUBLE, &startTime );
tstopAttr.write( H5::PredType::NATIVE_DOUBLE, &endTime );
dtAttr.write( H5::PredType::NATIVE_DOUBLE, ×tep );
detail::addStringAttribute( dataset, dataAttributes[4], _dunit );
detail::addStringAttribute( dataset, dataAttributes[5], _tunit );
}
void WriteTStrAttrib(const std::string& name, const std::string& value){
std::string str = value;
str.resize( STRING_ATTRIB_SIZE );
//define the data type as a string with the value's length
H5::StrType strType(0, STRING_ATTRIB_SIZE );
//open the data set and create a new attribute of type string
H5::Attribute attrib =
dSet_.createAttribute(name, strType, H5::DataSpace(H5S_SCALAR));
//write value to the attribute and close
attrib.write(strType, str );
attrib.close();
}
void TChainWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& chain, const std::string& meta) {
H5::H5File* h5file = H5Utils::openFile(fname);
H5::Group* h5group = H5Utils::openGroup(h5file, group);
// Dataset properties: optimized for reading/writing entire buffer at once
int rank = 3;
hsize_t dim[3] = {length_, nSamples_+2, nDim_};
H5::DataSpace dspace(rank, &(dim[0]));
H5::DSetCreatPropList plist;
plist.setDeflate(9); // gzip compression level
plist.setChunk(rank, &(dim[0]));
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
H5::DataSet* dataset = NULL;
try {
dataset = new H5::DataSet(h5group->createDataSet(chain, H5::PredType::NATIVE_FLOAT, dspace, plist));
} catch(H5::GroupIException &group_exception) {
std::cerr << "Could not create dataset for chain." << std::endl;
std::cerr << "Dataset '" << group << "/" << chain << "' most likely already exists." << std::endl;
throw;
}
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
if(meta == "") { // Store metadata as attributes
bool *converged = new bool[length_];
float *lnZ = new float[length_];
for(unsigned int i=0; i<length_; i++) {
converged[i] = metadata[i].converged;
lnZ[i] = metadata[i].lnZ;
}
// Allow large attributes to be stored in dense storage, versus compact (which has 64 kB limit)
//if(length_ > 5) {
// hid_t dID = dataset->getCreatePlist().getId();
// herr_t res = H5Pset_attr_phase_change(dID, 0, 0);
// std::cerr << res << std::endl;
// if(res < 0) {
// std::cerr << "Failed to specify dense storage." << std::endl;
// }
//}
H5::DataSpace convSpace(1, &(dim[0]));
H5::Attribute convAtt = dataset->createAttribute("converged", H5::PredType::NATIVE_CHAR, convSpace);
convAtt.write(H5::PredType::NATIVE_CHAR, reinterpret_cast<char*>(converged));
H5::DataSpace lnZSpace(1, &(dim[0]));
H5::Attribute lnZAtt = dataset->createAttribute("ln(Z)", H5::PredType::NATIVE_FLOAT, lnZSpace);
lnZAtt.write(H5::PredType::NATIVE_FLOAT, lnZ);
delete[] converged;
delete[] lnZ;
} else { // Store metadata as separate dataset
H5::CompType metaType(sizeof(TChainMetadata));
metaType.insertMember("converged", HOFFSET(TChainMetadata, converged), H5::PredType::NATIVE_CHAR);
metaType.insertMember("ln(Z)", HOFFSET(TChainMetadata, lnZ), H5::PredType::NATIVE_FLOAT);
rank = 1;
H5::DataSpace metaSpace(rank, &(dim[0]));
H5::DSetCreatPropList metaProp;
TChainMetadata emptyMetadata = {0, 0};
metaProp.setFillValue(metaType, &emptyMetadata);
metaProp.setDeflate(9);
metaProp.setChunk(rank, &(dim[0]));
H5::DataSet* metaDataset = new H5::DataSet(h5group->createDataSet(meta, metaType, metaSpace, metaProp));
metaDataset->write(metadata.data(), metaType);
delete metaDataset;
metaDataset = NULL;
}
delete dataset;
delete h5group;
delete h5file;
//std::cerr << "Cleaned up." << std::endl;
}
bool save_mat_image(cv::Mat& img, TRect& rect, std::string fname, std::string group_name,
std::string dset_name, std::string dim1, std::string dim2, int compression) {
assert((img.dims == 2) && (img.rows == rect.N_bins[0]) && (img.cols == rect.N_bins[1]));
if((compression<0) || (compression > 9)) {
std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
return false;
}
H5::Exception::dontPrint();
H5::H5File *file = H5Utils::openFile(fname);
if(file == NULL) { return false; }
H5::Group *group = H5Utils::openGroup(file, group_name);
if(group == NULL) {
delete file;
return false;
}
/*
* Image Data
*/
// Creation property list
H5::DSetCreatPropList plist;
int rank = 2;
hsize_t dim[2] = {rect.N_bins[0], rect.N_bins[1]};
plist.setDeflate(compression); // gzip compression level
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
plist.setChunk(rank, &(dim[0]));
H5::DataSpace dspace(rank, &(dim[0]));
H5::DataSet* dataset;
try {
dataset = new H5::DataSet(group->createDataSet(dset_name, H5::PredType::NATIVE_FLOAT, dspace, plist));
} catch(H5::FileIException create_dset_err) {
std::cerr << "Unable to create dataset '" << dset_name << "'." << std::endl;
delete group;
delete file;
return false;
}
float *buf = new float[rect.N_bins[0]*rect.N_bins[1]];
for(size_t j=0; j<rect.N_bins[0]; j++) {
for(size_t k=0; k<rect.N_bins[1]; k++) {
buf[rect.N_bins[1]*j + k] = img.at<double>(j,k);
/*float tmp = img.at<double>(j,k);
if(tmp > 0.) {
std::cerr << j << ", " << k << " --> " << j + rect.N_bins[0]*k << " --> " << tmp << std::endl;
}*/
}
}
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
/*
* Attributes
*/
hsize_t att_dim = 2;
H5::DataSpace att_dspace(1, &att_dim);
H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
H5::Attribute att_N = dataset->createAttribute("N_pix", att_dtype, att_dspace);
att_N.write(att_dtype, &(rect.N_bins));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
att_min.write(att_dtype, &(rect.min));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
att_max.write(att_dtype, &(rect.max));
att_dim = 1;
H5::StrType vls_type(0, H5T_VARIABLE);
H5::DataSpace att_space_str(H5S_SCALAR);
H5::Attribute att_name_1 = dataset->createAttribute("dim_name_1", vls_type, att_space_str);
att_name_1.write(vls_type, dim1);
H5::Attribute att_name_2 = dataset->createAttribute("dim_name_2", vls_type, att_space_str);
att_name_2.write(vls_type, dim2);
file->close();
delete[] buf;
delete dataset;
delete group;
delete file;
return true;
}
bool TStellarData::save(const std::string& fname, const std::string& group, const std::string &dset, int compression) {
if((compression < 0) || (compression > 9)) {
std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
return false;
}
hsize_t nstars = star.size();
if(nstars == 0) {
std::cerr << "! No stars to write." << std::endl;
return false;
}
H5::Exception::dontPrint();
H5::H5File *file = H5Utils::openFile(fname);
if(file == NULL) { return false; }
H5::Group *gp = H5Utils::openGroup(file, group);
if(gp == NULL) {
delete file;
return false;
}
/*
* Photometry
*/
// Datatype
hsize_t nbands = NBANDS;
H5::ArrayType f4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
H5::ArrayType u4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
H5::CompType dtype(sizeof(TFileData));
dtype.insertMember("obj_id", HOFFSET(TFileData, obj_id), H5::PredType::NATIVE_UINT64);
dtype.insertMember("l", HOFFSET(TFileData, l), H5::PredType::NATIVE_DOUBLE);
dtype.insertMember("b", HOFFSET(TFileData, b), H5::PredType::NATIVE_DOUBLE);
dtype.insertMember("mag", HOFFSET(TFileData, mag), f4arr);
dtype.insertMember("err", HOFFSET(TFileData, err), f4arr);
dtype.insertMember("maglimit", HOFFSET(TFileData, maglimit), f4arr);
dtype.insertMember("nDet", HOFFSET(TFileData, N_det), u4arr);
dtype.insertMember("EBV", HOFFSET(TFileData, EBV), H5::PredType::NATIVE_FLOAT);
// Dataspace
hsize_t dim = nstars;
H5::DataSpace dspace(1, &dim);
// Property List
H5::DSetCreatPropList plist;
plist.setChunk(1, &nstars);
plist.setDeflate(compression);
// Dataset
H5::DataSet dataset = gp->createDataSet(dset, dtype, dspace, plist);
// Write dataset
TFileData* data = new TFileData[nstars];
for(size_t i=0; i<nstars; i++) {
data[i].obj_id = star[i].obj_id;
data[i].l = star[i].l;
data[i].b = star[i].b;
for(size_t k=0; k<NBANDS; k++) {
data[i].mag[k] = star[i].m[k];
data[i].err[k] = star[i].err[k];
data[i].maglimit[k] = star[i].maglimit[k];
}
data[i].EBV = star[i].EBV;
}
dataset.write(data, dtype);
/*
* Attributes
*/
dim = 1;
H5::DataSpace att_dspace(1, &dim);
H5::PredType att_dtype = H5::PredType::NATIVE_UINT64;
H5::Attribute att_healpix_index = dataset.createAttribute("healpix_index", att_dtype, att_dspace);
att_healpix_index.write(att_dtype, &healpix_index);
att_dtype = H5::PredType::NATIVE_UINT32;
H5::Attribute att_nside = dataset.createAttribute("nside", att_dtype, att_dspace);
att_nside.write(att_dtype, &nside);
att_dtype = H5::PredType::NATIVE_UCHAR;
H5::Attribute att_nested = dataset.createAttribute("nested", att_dtype, att_dspace);
att_nested.write(att_dtype, &nested);
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_l = dataset.createAttribute("l", att_dtype, att_dspace);
att_l.write(att_dtype, &l);
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_b = dataset.createAttribute("b", att_dtype, att_dspace);
att_b.write(att_dtype, &b);
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_EBV = dataset.createAttribute("EBV", att_dtype, att_dspace);
att_EBV.write(att_dtype, &EBV);
file->close();
delete[] data;
delete gp;
delete file;
return true;
}
void Bundle2::initFrameStream_() {
H5::DataSpace scalar;
// Creating datasets for each frame
H5::Group bundleGroup = streamFile_->createGroup("/Bundle");
H5::Group framesGroup = bundleGroup.createGroup("Frames");
hsize_t count = frames_.size();
H5::Attribute attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
// Defining frame dataset property
hsize_t chunkDim[] = { 1, 2, 10 };
H5::DSetCreatPropList propList;
propList.setLayout(H5D_CHUNKED);
propList.setChunk(3, chunkDim);
propList.setDeflate(9);
// Definig dataset dataspace
hsize_t max_dim[] = { numCameras_, 2, H5S_UNLIMITED };
hsize_t dim[] = { numCameras_, 2, 0 };
H5::DataSpace ds(3, dim, max_dim);
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
const std::string datasetName = boost::str(boost::format("Frame %1$04d") % (*it)->number());
// Creating dataset
H5::DataSet frameData = framesGroup.createDataSet(datasetName, H5::PredType::IEEE_F32LE, ds, propList);
// Writing global number
attr = frameData.createAttribute("globalNumber", H5::PredType::STD_U64LE, scalar);
count = (*it)->globalNumber();
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
// Clean up!
frameData.close();
}
// Clean up!
ds.close();
propList.close();
// Creating tracks dataset
hsize_t chunkDim2[] = { 2, 10 };
propList = H5::DSetCreatPropList();
propList.setLayout(H5D_CHUNKED);
propList.setChunk(2, chunkDim);
propList.setDeflate(9);
H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE);
hsize_t tracksDim[] = { 0, 2 };
hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 };
H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim);
H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList);
tracksDataset.close();
tracksDataspace.close();
tracksDatasetType.close();
propList.close();
framesGroup.close();
bundleGroup.close();
scalar.close();
}