void TImgWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& img) {
H5::H5File* h5file = H5Utils::openFile(fname);
H5::Group* h5group = H5Utils::openGroup(h5file, group);
// Dataset properties: optimized for reading/writing entire buffer at once
int rank = 3;
hsize_t dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
hsize_t chunk_dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
if(length_ > 1000) {
float div = ceil((float)length_ / 1000.);
chunk_dim[0] = (int)ceil(length_ / div);
std::cerr << "! Changing chunk length to " << chunk_dim[0] << " stars." << std::endl;
}
H5::DataSpace dspace(rank, &(dim[0]));
H5::DSetCreatPropList plist;
plist.setDeflate(9); // gzip compression level
plist.setChunk(rank, &(chunk_dim[0]));
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
H5::DataSet* dataset = new H5::DataSet(h5group->createDataSet(img, H5::PredType::NATIVE_FLOAT, dspace, plist));
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
/*
* Attributes
*/
hsize_t att_dim = 2;
H5::DataSpace att_dspace(1, &att_dim);
H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
H5::Attribute att_N = dataset->createAttribute("nPix", att_dtype, att_dspace);
att_N.write(att_dtype, &(rect_.N_bins));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
att_min.write(att_dtype, &(rect_.min));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
att_max.write(att_dtype, &(rect_.max));
delete dataset;
delete h5group;
delete h5file;
}
void TChainWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& chain, const std::string& meta) {
H5::H5File* h5file = H5Utils::openFile(fname);
H5::Group* h5group = H5Utils::openGroup(h5file, group);
// Dataset properties: optimized for reading/writing entire buffer at once
int rank = 3;
hsize_t dim[3] = {length_, nSamples_+2, nDim_};
H5::DataSpace dspace(rank, &(dim[0]));
H5::DSetCreatPropList plist;
plist.setDeflate(9); // gzip compression level
plist.setChunk(rank, &(dim[0]));
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
H5::DataSet* dataset = NULL;
try {
dataset = new H5::DataSet(h5group->createDataSet(chain, H5::PredType::NATIVE_FLOAT, dspace, plist));
} catch(H5::GroupIException &group_exception) {
std::cerr << "Could not create dataset for chain." << std::endl;
std::cerr << "Dataset '" << group << "/" << chain << "' most likely already exists." << std::endl;
throw;
}
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
if(meta == "") { // Store metadata as attributes
bool *converged = new bool[length_];
float *lnZ = new float[length_];
for(unsigned int i=0; i<length_; i++) {
converged[i] = metadata[i].converged;
lnZ[i] = metadata[i].lnZ;
}
// Allow large attributes to be stored in dense storage, versus compact (which has 64 kB limit)
//if(length_ > 5) {
// hid_t dID = dataset->getCreatePlist().getId();
// herr_t res = H5Pset_attr_phase_change(dID, 0, 0);
// std::cerr << res << std::endl;
// if(res < 0) {
// std::cerr << "Failed to specify dense storage." << std::endl;
// }
//}
H5::DataSpace convSpace(1, &(dim[0]));
H5::Attribute convAtt = dataset->createAttribute("converged", H5::PredType::NATIVE_CHAR, convSpace);
convAtt.write(H5::PredType::NATIVE_CHAR, reinterpret_cast<char*>(converged));
H5::DataSpace lnZSpace(1, &(dim[0]));
H5::Attribute lnZAtt = dataset->createAttribute("ln(Z)", H5::PredType::NATIVE_FLOAT, lnZSpace);
lnZAtt.write(H5::PredType::NATIVE_FLOAT, lnZ);
delete[] converged;
delete[] lnZ;
} else { // Store metadata as separate dataset
H5::CompType metaType(sizeof(TChainMetadata));
metaType.insertMember("converged", HOFFSET(TChainMetadata, converged), H5::PredType::NATIVE_CHAR);
metaType.insertMember("ln(Z)", HOFFSET(TChainMetadata, lnZ), H5::PredType::NATIVE_FLOAT);
rank = 1;
H5::DataSpace metaSpace(rank, &(dim[0]));
H5::DSetCreatPropList metaProp;
TChainMetadata emptyMetadata = {0, 0};
metaProp.setFillValue(metaType, &emptyMetadata);
metaProp.setDeflate(9);
metaProp.setChunk(rank, &(dim[0]));
H5::DataSet* metaDataset = new H5::DataSet(h5group->createDataSet(meta, metaType, metaSpace, metaProp));
metaDataset->write(metadata.data(), metaType);
delete metaDataset;
metaDataset = NULL;
}
delete dataset;
delete h5group;
delete h5file;
//std::cerr << "Cleaned up." << std::endl;
}
bool TChain::save(std::string fname, std::string group_name, size_t index,
std::string dim_name, int compression, int subsample,
bool converged, float lnZ) const {
if((compression<0) || (compression > 9)) {
std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
return false;
}
H5::Exception::dontPrint();
H5::H5File *file = H5Utils::openFile(fname);
if(file == NULL) { return false; }
/*
try {
file->unlink(group_name);
} catch(...) {
// pass
}
*/
H5::Group *group = H5Utils::openGroup(file, group_name);
if(group == NULL) {
delete file;
return false;
}
/*
* Attributes
*/
// Datatype
H5::CompType att_type(sizeof(TChainAttribute));
hid_t tid = H5Tcopy(H5T_C_S1);
H5Tset_size(tid, H5T_VARIABLE);
att_type.insertMember("dim_name", HOFFSET(TChainAttribute, dim_name), tid);
//att_type.insertMember("total_weight", HOFFSET(TChainAttribute, total_weight), H5::PredType::NATIVE_FLOAT);
//att_type.insertMember("ndim", HOFFSET(TChainAttribute, ndim), H5::PredType::NATIVE_UINT64);
//att_type.insertMember("length", HOFFSET(TChainAttribute, length), H5::PredType::NATIVE_UINT64);
// Dataspace
int att_rank = 1;
hsize_t att_dim = 1;
H5::DataSpace att_space(att_rank, &att_dim);
// Dataset
//H5::Attribute att = group->createAttribute("parameter names", att_type, att_space);
TChainAttribute att_data;
att_data.dim_name = new char[dim_name.size()+1];
std::strcpy(att_data.dim_name, dim_name.c_str());
//att_data.total_weight = total_weight;
//att_data.ndim = N;
//att_data.length = length;
//att.write(att_type, &att_data);
delete[] att_data.dim_name;
//int att_rank = 1;
//hsize_t att_dim = 1;
H5::DataType conv_dtype = H5::PredType::NATIVE_UCHAR;
H5::DataSpace conv_dspace(att_rank, &att_dim);
//H5::Attribute conv_att = H5Utils::openAttribute(group, "converged", conv_dtype, conv_dspace);
//conv_att.write(conv_dtype, &converged);
H5::DataType lnZ_dtype = H5::PredType::NATIVE_FLOAT;
H5::DataSpace lnZ_dspace(att_rank, &att_dim);
//H5::Attribute lnZ_att = H5Utils::openAttribute(group, "ln Z", lnZ_dtype, lnZ_dspace);
//lnZ_att.write(lnZ_dtype, &lnZ);
// Creation property list to be used for all three datasets
H5::DSetCreatPropList plist;
//plist.setDeflate(compression); // gzip compression level
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
H5D_layout_t layout = H5D_COMPACT;
plist.setLayout(layout);
/*
* Choose subsample of points in chain
*/
size_t *el_idx = NULL;
size_t *subsample_idx = NULL;
if(subsample > 0) {
size_t tot_weight_tmp = (size_t)ceil(total_weight);
el_idx = new size_t[tot_weight_tmp];
size_t unrolled_idx = 0;
size_t chain_idx = 0;
std::vector<double>::const_iterator it, it_end;
it_end = w.end();
for(it = w.begin(); it != it_end; ++it, chain_idx++) {
for(size_t n = unrolled_idx; n < unrolled_idx + (size_t)(*it); n++) {
el_idx[n] = chain_idx;
}
unrolled_idx += (size_t)(*it);
}
assert(chain_idx == length);
gsl_rng *r;
seed_gsl_rng(&r);
subsample_idx = new size_t[tot_weight_tmp];
for(size_t i=0; i<subsample; i++) {
subsample_idx[i] = el_idx[gsl_rng_uniform_int(r, tot_weight_tmp)];
}
}
/*
* Coordinates
*/
// Dataspace
hsize_t dim;
if(subsample > 0) {
dim = subsample;
} else {
dim = length;
}
// Chunking (required for compression)
int rank = 2;
hsize_t coord_dim[2] = {dim, N};
//if(dim < chunk) {
//plist.setChunk(rank, &(coord_dim[0]));
//} else {
// plist.setChunk(rank, &chunk);
//}
H5::DataSpace x_dspace(rank, &(coord_dim[0]));
// Dataset
//std::stringstream x_dset_path;
//x_dset_path << group_name << "/chain/coords";
std::stringstream coordname;
coordname << "coords " << index;
H5::DataSet* x_dataset = new H5::DataSet(group->createDataSet(coordname.str(), H5::PredType::NATIVE_FLOAT, x_dspace, plist));
// Write
float *buf = new float[N*dim];
if(subsample > 0) {
size_t tmp_idx;
for(size_t i=0; i<subsample; i++) {
tmp_idx = subsample_idx[i];
for(size_t k=0; k<N; k++) {
buf[N*i+k] = x[N*tmp_idx+k];
}
}
} else {
for(size_t i=0; i<dim; i++) { buf[i] = x[i]; }
}
x_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
/*
* Weights
*/
// Dataspace
if(subsample <= 0) {
dim = w.size();
rank = 1;
H5::DataSpace w_dspace(rank, &dim);
// Dataset
//std::stringstream w_dset_path;
//w_dset_path << group_name << "/chain/weights";
H5::DataSet* w_dataset = new H5::DataSet(group->createDataSet("weights", H5::PredType::NATIVE_FLOAT, w_dspace, plist));
// Write
if(subsample > 0) {
for(size_t i=0; i<subsample; i++) { buf[i] = 1.; }
} else {
assert(w.size() < x.size());
for(size_t i=0; i<w.size(); i++) { buf[i] = w[i]; }
}
w_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
delete w_dataset;
}
/*
* Probability densities
*/
// Dataspace
rank = 1;
H5::DataSpace L_dspace(rank, &dim);
// Dataset
//std::stringstream L_dset_path;
//L_dset_path << group_name << "/chain/probs";
std::stringstream lnpname;
lnpname << "ln_p " << index;
H5::DataSet* L_dataset = new H5::DataSet(group->createDataSet(lnpname.str(), H5::PredType::NATIVE_FLOAT, L_dspace, plist));
// Write
if(subsample > 0) {
for(size_t i=0; i<subsample; i++) { buf[i] = L[subsample_idx[i]]; }
} else {
assert(L.size() < x.size());
for(size_t i=0; i<L.size(); i++) { buf[i] = L[i]; }
}
L_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
if(subsample > 0) {
delete[] el_idx;
delete[] subsample_idx;
}
delete[] buf;
delete x_dataset;
delete L_dataset;
delete group;
delete file;
return true;
}
bool save_mat_image(cv::Mat& img, TRect& rect, std::string fname, std::string group_name,
std::string dset_name, std::string dim1, std::string dim2, int compression) {
assert((img.dims == 2) && (img.rows == rect.N_bins[0]) && (img.cols == rect.N_bins[1]));
if((compression<0) || (compression > 9)) {
std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
return false;
}
H5::Exception::dontPrint();
H5::H5File *file = H5Utils::openFile(fname);
if(file == NULL) { return false; }
H5::Group *group = H5Utils::openGroup(file, group_name);
if(group == NULL) {
delete file;
return false;
}
/*
* Image Data
*/
// Creation property list
H5::DSetCreatPropList plist;
int rank = 2;
hsize_t dim[2] = {rect.N_bins[0], rect.N_bins[1]};
plist.setDeflate(compression); // gzip compression level
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
plist.setChunk(rank, &(dim[0]));
H5::DataSpace dspace(rank, &(dim[0]));
H5::DataSet* dataset;
try {
dataset = new H5::DataSet(group->createDataSet(dset_name, H5::PredType::NATIVE_FLOAT, dspace, plist));
} catch(H5::FileIException create_dset_err) {
std::cerr << "Unable to create dataset '" << dset_name << "'." << std::endl;
delete group;
delete file;
return false;
}
float *buf = new float[rect.N_bins[0]*rect.N_bins[1]];
for(size_t j=0; j<rect.N_bins[0]; j++) {
for(size_t k=0; k<rect.N_bins[1]; k++) {
buf[rect.N_bins[1]*j + k] = img.at<double>(j,k);
/*float tmp = img.at<double>(j,k);
if(tmp > 0.) {
std::cerr << j << ", " << k << " --> " << j + rect.N_bins[0]*k << " --> " << tmp << std::endl;
}*/
}
}
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
/*
* Attributes
*/
hsize_t att_dim = 2;
H5::DataSpace att_dspace(1, &att_dim);
H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
H5::Attribute att_N = dataset->createAttribute("N_pix", att_dtype, att_dspace);
att_N.write(att_dtype, &(rect.N_bins));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
att_min.write(att_dtype, &(rect.min));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
att_max.write(att_dtype, &(rect.max));
att_dim = 1;
H5::StrType vls_type(0, H5T_VARIABLE);
H5::DataSpace att_space_str(H5S_SCALAR);
H5::Attribute att_name_1 = dataset->createAttribute("dim_name_1", vls_type, att_space_str);
att_name_1.write(vls_type, dim1);
H5::Attribute att_name_2 = dataset->createAttribute("dim_name_2", vls_type, att_space_str);
att_name_2.write(vls_type, dim2);
file->close();
delete[] buf;
delete dataset;
delete group;
delete file;
return true;
}