h5d::dataset::ptr_type create_dataset(std::string const & name, std::vector<MemType> const & data, int compression=0) { if (H5Lexists(_self, name.c_str(), H5P_DEFAULT) > 0) throw Exception("Object already exists with that name"); h5t::wrapper<StorageType> t; h5t::datatype type(t); h5s::dataspace dspace(std::vector<hsize_t>(1,data.size()), std::vector<hsize_t>(1,H5S_UNLIMITED)); h5d::dataset::ptr_type ds = boost::make_shared<h5d::dataset>(_self, name.c_str(), dspace, type, compression); ds->write(data); return ds; }
void TImgWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& img) { H5::H5File* h5file = H5Utils::openFile(fname); H5::Group* h5group = H5Utils::openGroup(h5file, group); // Dataset properties: optimized for reading/writing entire buffer at once int rank = 3; hsize_t dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]}; hsize_t chunk_dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]}; if(length_ > 1000) { float div = ceil((float)length_ / 1000.); chunk_dim[0] = (int)ceil(length_ / div); std::cerr << "! Changing chunk length to " << chunk_dim[0] << " stars." << std::endl; } H5::DataSpace dspace(rank, &(dim[0])); H5::DSetCreatPropList plist; plist.setDeflate(9); // gzip compression level plist.setChunk(rank, &(chunk_dim[0])); float fillvalue = 0; plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue); H5::DataSet* dataset = new H5::DataSet(h5group->createDataSet(img, H5::PredType::NATIVE_FLOAT, dspace, plist)); dataset->write(buf, H5::PredType::NATIVE_FLOAT); /* * Attributes */ hsize_t att_dim = 2; H5::DataSpace att_dspace(1, &att_dim); H5::PredType att_dtype = H5::PredType::NATIVE_UINT32; H5::Attribute att_N = dataset->createAttribute("nPix", att_dtype, att_dspace); att_N.write(att_dtype, &(rect_.N_bins)); att_dtype = H5::PredType::NATIVE_DOUBLE; H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace); att_min.write(att_dtype, &(rect_.min)); att_dtype = H5::PredType::NATIVE_DOUBLE; H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace); att_max.write(att_dtype, &(rect_.max)); delete dataset; delete h5group; delete h5file; }
/* only for Eigen3 matrix/vector */ void HDF5IO::saveVector(const std::string& GroupName, const std::string& Name, const RealVectorType& V) { hsize_t Dim[1] = {hsize_t(V.size())}; H5::DataSpace dspace(1,Dim); H5::Group FG = getGroup( GroupName ); try{ H5::Exception::dontPrint(); H5::DataSet DataSet = FG.openDataSet(Name.c_str()); DataSet.write(V.data(),H5::PredType::NATIVE_DOUBLE, dspace); } catch ( const H5::GroupIException not_found_error ){ H5::DataSet DataSet = FG.createDataSet(Name.c_str(), H5::PredType::NATIVE_DOUBLE,dspace); DataSet.write(V.data(),H5::PredType::NATIVE_DOUBLE); } FG.close(); }
static void save_molecular_type(const MolecularTypeBase* mtb, std::vector<std::pair<ParticleID, Voxel> > voxels, H5::Group* group) { const Species species(mtb->species()); boost::scoped_ptr<H5::Group> mtgroup( new H5::Group(group->createGroup(species.serial().c_str()))); h5_species_struct property; property.radius = mtb->radius(); property.D = mtb->D(); const MolecularTypeBase* loc(mtb->location()); if (loc->is_vacant()) std::strcpy(property.location, ""); else std::strcpy(property.location, loc->species().serial().c_str()); property.is_structure = mtb->is_structure() ? 1 : 0; property.dimension = mtb->get_dimension(); H5::CompType property_comp_type(get_property_comp()); mtgroup->createAttribute("property", property_comp_type, H5::DataSpace(H5S_SCALAR)).write(property_comp_type, &property); // Save voxels const Integer num_voxels(voxels.size()); std::size_t vidx(0); boost::scoped_array<h5_voxel_struct> h5_voxel_array(new h5_voxel_struct[num_voxels]); for (std::vector<std::pair<ParticleID, Voxel> >::const_iterator itr(voxels.begin()); itr != voxels.end(); ++itr) { h5_voxel_array[vidx].lot = (*itr).first.lot(); h5_voxel_array[vidx].serial = (*itr).first.serial(); h5_voxel_array[vidx].coordinate = (*itr).second.coordinate(); ++vidx; } H5::CompType voxel_comp_type(get_voxel_comp()); hsize_t dims[] = {num_voxels}; H5::DataSpace dspace(/* RANK= */1, dims); boost::scoped_ptr<H5::DataSet> dset(new H5::DataSet( mtgroup->createDataSet("voxels", voxel_comp_type, dspace))); dset->write(h5_voxel_array.get(), dset->getDataType()); }
void HDF5IO::saveStdVector(const std::string& GroupName, const std::string& Name, const std::vector<size_t>& V) { try{ hsize_t Dim[1] = {hsize_t(V.size())}; H5::DataSpace dspace(1,Dim); H5::Group FG = getGroup( GroupName ); try{ H5::Exception::dontPrint(); H5::DataSet DataSet = FG.openDataSet(Name.c_str()); DataSet.write(V.data(),H5::PredType::NATIVE_ULONG, dspace); } catch ( const H5::GroupIException not_found_error ){ H5::DataSet DataSet = FG.createDataSet(Name.c_str(), H5::PredType::NATIVE_ULONG, dspace); DataSet.write(V.data(), H5::PredType::NATIVE_ULONG); } FG.close(); } catch( const H5::Exception err ){ RUNTIME_ERROR("HDF5IO::saveUlongStdVector "); } }
void HDF5IO::saveVector(const std::string& GroupName, const std::string& Name, const ComplexVectorType& V) { try{ H5::CompType ComplexDataType = this->openCompType("complex"); hsize_t Dim[1] = {hsize_t(V.size())}; H5::DataSpace dspace(1,Dim); H5::Group FG = getGroup( GroupName ); try{ H5::Exception::dontPrint(); H5::DataSet DataSet = FG.openDataSet(Name.c_str()); DataSet.write(V.data(), ComplexDataType, dspace); } catch ( const H5::GroupIException not_found_error ){ H5::DataSet DataSet = FG.createDataSet(Name.c_str(), ComplexDataType, dspace); DataSet.write(V.data(), ComplexDataType); } FG.close(); } catch ( const H5::DataSetIException error ){ error.printError(); RUNTIME_ERROR("HDF5IO::saveComplexVector at "); } }
void TChainWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& chain, const std::string& meta) { H5::H5File* h5file = H5Utils::openFile(fname); H5::Group* h5group = H5Utils::openGroup(h5file, group); // Dataset properties: optimized for reading/writing entire buffer at once int rank = 3; hsize_t dim[3] = {length_, nSamples_+2, nDim_}; H5::DataSpace dspace(rank, &(dim[0])); H5::DSetCreatPropList plist; plist.setDeflate(9); // gzip compression level plist.setChunk(rank, &(dim[0])); float fillvalue = 0; plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue); H5::DataSet* dataset = NULL; try { dataset = new H5::DataSet(h5group->createDataSet(chain, H5::PredType::NATIVE_FLOAT, dspace, plist)); } catch(H5::GroupIException &group_exception) { std::cerr << "Could not create dataset for chain." << std::endl; std::cerr << "Dataset '" << group << "/" << chain << "' most likely already exists." << std::endl; throw; } dataset->write(buf, H5::PredType::NATIVE_FLOAT); if(meta == "") { // Store metadata as attributes bool *converged = new bool[length_]; float *lnZ = new float[length_]; for(unsigned int i=0; i<length_; i++) { converged[i] = metadata[i].converged; lnZ[i] = metadata[i].lnZ; } // Allow large attributes to be stored in dense storage, versus compact (which has 64 kB limit) //if(length_ > 5) { // hid_t dID = dataset->getCreatePlist().getId(); // herr_t res = H5Pset_attr_phase_change(dID, 0, 0); // std::cerr << res << std::endl; // if(res < 0) { // std::cerr << "Failed to specify dense storage." << std::endl; // } //} H5::DataSpace convSpace(1, &(dim[0])); H5::Attribute convAtt = dataset->createAttribute("converged", H5::PredType::NATIVE_CHAR, convSpace); convAtt.write(H5::PredType::NATIVE_CHAR, reinterpret_cast<char*>(converged)); H5::DataSpace lnZSpace(1, &(dim[0])); H5::Attribute lnZAtt = dataset->createAttribute("ln(Z)", H5::PredType::NATIVE_FLOAT, lnZSpace); lnZAtt.write(H5::PredType::NATIVE_FLOAT, lnZ); delete[] converged; delete[] lnZ; } else { // Store metadata as separate dataset H5::CompType metaType(sizeof(TChainMetadata)); metaType.insertMember("converged", HOFFSET(TChainMetadata, converged), H5::PredType::NATIVE_CHAR); metaType.insertMember("ln(Z)", HOFFSET(TChainMetadata, lnZ), H5::PredType::NATIVE_FLOAT); rank = 1; H5::DataSpace metaSpace(rank, &(dim[0])); H5::DSetCreatPropList metaProp; TChainMetadata emptyMetadata = {0, 0}; metaProp.setFillValue(metaType, &emptyMetadata); metaProp.setDeflate(9); metaProp.setChunk(rank, &(dim[0])); H5::DataSet* metaDataset = new H5::DataSet(h5group->createDataSet(meta, metaType, metaSpace, metaProp)); metaDataset->write(metadata.data(), metaType); delete metaDataset; metaDataset = NULL; } delete dataset; delete h5group; delete h5file; //std::cerr << "Cleaned up." << std::endl; }
bool save_mat_image(cv::Mat& img, TRect& rect, std::string fname, std::string group_name, std::string dset_name, std::string dim1, std::string dim2, int compression) { assert((img.dims == 2) && (img.rows == rect.N_bins[0]) && (img.cols == rect.N_bins[1])); if((compression<0) || (compression > 9)) { std::cerr << "! Invalid gzip compression level: " << compression << std::endl; return false; } H5::Exception::dontPrint(); H5::H5File *file = H5Utils::openFile(fname); if(file == NULL) { return false; } H5::Group *group = H5Utils::openGroup(file, group_name); if(group == NULL) { delete file; return false; } /* * Image Data */ // Creation property list H5::DSetCreatPropList plist; int rank = 2; hsize_t dim[2] = {rect.N_bins[0], rect.N_bins[1]}; plist.setDeflate(compression); // gzip compression level float fillvalue = 0; plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue); plist.setChunk(rank, &(dim[0])); H5::DataSpace dspace(rank, &(dim[0])); H5::DataSet* dataset; try { dataset = new H5::DataSet(group->createDataSet(dset_name, H5::PredType::NATIVE_FLOAT, dspace, plist)); } catch(H5::FileIException create_dset_err) { std::cerr << "Unable to create dataset '" << dset_name << "'." << std::endl; delete group; delete file; return false; } float *buf = new float[rect.N_bins[0]*rect.N_bins[1]]; for(size_t j=0; j<rect.N_bins[0]; j++) { for(size_t k=0; k<rect.N_bins[1]; k++) { buf[rect.N_bins[1]*j + k] = img.at<double>(j,k); /*float tmp = img.at<double>(j,k); if(tmp > 0.) { std::cerr << j << ", " << k << " --> " << j + rect.N_bins[0]*k << " --> " << tmp << std::endl; }*/ } } dataset->write(buf, H5::PredType::NATIVE_FLOAT); /* * Attributes */ hsize_t att_dim = 2; H5::DataSpace att_dspace(1, &att_dim); H5::PredType att_dtype = H5::PredType::NATIVE_UINT32; H5::Attribute att_N = dataset->createAttribute("N_pix", att_dtype, att_dspace); att_N.write(att_dtype, &(rect.N_bins)); att_dtype = H5::PredType::NATIVE_DOUBLE; H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace); att_min.write(att_dtype, &(rect.min)); att_dtype = H5::PredType::NATIVE_DOUBLE; H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace); att_max.write(att_dtype, &(rect.max)); att_dim = 1; H5::StrType vls_type(0, H5T_VARIABLE); H5::DataSpace att_space_str(H5S_SCALAR); H5::Attribute att_name_1 = dataset->createAttribute("dim_name_1", vls_type, att_space_str); att_name_1.write(vls_type, dim1); H5::Attribute att_name_2 = dataset->createAttribute("dim_name_2", vls_type, att_space_str); att_name_2.write(vls_type, dim2); file->close(); delete[] buf; delete dataset; delete group; delete file; return true; }
bool TStellarData::save(const std::string& fname, const std::string& group, const std::string &dset, int compression) { if((compression < 0) || (compression > 9)) { std::cerr << "! Invalid gzip compression level: " << compression << std::endl; return false; } hsize_t nstars = star.size(); if(nstars == 0) { std::cerr << "! No stars to write." << std::endl; return false; } H5::Exception::dontPrint(); H5::H5File *file = H5Utils::openFile(fname); if(file == NULL) { return false; } H5::Group *gp = H5Utils::openGroup(file, group); if(gp == NULL) { delete file; return false; } /* * Photometry */ // Datatype hsize_t nbands = NBANDS; H5::ArrayType f4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands); H5::ArrayType u4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands); H5::CompType dtype(sizeof(TFileData)); dtype.insertMember("obj_id", HOFFSET(TFileData, obj_id), H5::PredType::NATIVE_UINT64); dtype.insertMember("l", HOFFSET(TFileData, l), H5::PredType::NATIVE_DOUBLE); dtype.insertMember("b", HOFFSET(TFileData, b), H5::PredType::NATIVE_DOUBLE); dtype.insertMember("mag", HOFFSET(TFileData, mag), f4arr); dtype.insertMember("err", HOFFSET(TFileData, err), f4arr); dtype.insertMember("maglimit", HOFFSET(TFileData, maglimit), f4arr); dtype.insertMember("nDet", HOFFSET(TFileData, N_det), u4arr); dtype.insertMember("EBV", HOFFSET(TFileData, EBV), H5::PredType::NATIVE_FLOAT); // Dataspace hsize_t dim = nstars; H5::DataSpace dspace(1, &dim); // Property List H5::DSetCreatPropList plist; plist.setChunk(1, &nstars); plist.setDeflate(compression); // Dataset H5::DataSet dataset = gp->createDataSet(dset, dtype, dspace, plist); // Write dataset TFileData* data = new TFileData[nstars]; for(size_t i=0; i<nstars; i++) { data[i].obj_id = star[i].obj_id; data[i].l = star[i].l; data[i].b = star[i].b; for(size_t k=0; k<NBANDS; k++) { data[i].mag[k] = star[i].m[k]; data[i].err[k] = star[i].err[k]; data[i].maglimit[k] = star[i].maglimit[k]; } data[i].EBV = star[i].EBV; } dataset.write(data, dtype); /* * Attributes */ dim = 1; H5::DataSpace att_dspace(1, &dim); H5::PredType att_dtype = H5::PredType::NATIVE_UINT64; H5::Attribute att_healpix_index = dataset.createAttribute("healpix_index", att_dtype, att_dspace); att_healpix_index.write(att_dtype, &healpix_index); att_dtype = H5::PredType::NATIVE_UINT32; H5::Attribute att_nside = dataset.createAttribute("nside", att_dtype, att_dspace); att_nside.write(att_dtype, &nside); att_dtype = H5::PredType::NATIVE_UCHAR; H5::Attribute att_nested = dataset.createAttribute("nested", att_dtype, att_dspace); att_nested.write(att_dtype, &nested); att_dtype = H5::PredType::NATIVE_DOUBLE; H5::Attribute att_l = dataset.createAttribute("l", att_dtype, att_dspace); att_l.write(att_dtype, &l); att_dtype = H5::PredType::NATIVE_DOUBLE; H5::Attribute att_b = dataset.createAttribute("b", att_dtype, att_dspace); att_b.write(att_dtype, &b); att_dtype = H5::PredType::NATIVE_DOUBLE; H5::Attribute att_EBV = dataset.createAttribute("EBV", att_dtype, att_dspace); att_EBV.write(att_dtype, &EBV); file->close(); delete[] data; delete gp; delete file; return true; }