示例#1
0
文件: h5g.hpp 项目: margoliashlab/arf
	h5d::dataset::ptr_type
	create_dataset(std::string const & name,
		       std::vector<MemType> const & data,
		       int compression=0) {
		if (H5Lexists(_self, name.c_str(), H5P_DEFAULT) > 0)
			throw Exception("Object already exists with that name");

		h5t::wrapper<StorageType> t;
		h5t::datatype type(t);
		h5s::dataspace dspace(std::vector<hsize_t>(1,data.size()), std::vector<hsize_t>(1,H5S_UNLIMITED));
		h5d::dataset::ptr_type ds = boost::make_shared<h5d::dataset>(_self, name.c_str(), dspace,
									     type, compression);
		ds->write(data);
		return ds;
	}
示例#2
0
void TImgWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& img) {
	H5::H5File* h5file = H5Utils::openFile(fname);
	H5::Group* h5group = H5Utils::openGroup(h5file, group);
	
	// Dataset properties: optimized for reading/writing entire buffer at once
	int rank = 3;
	hsize_t dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
	hsize_t chunk_dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
	if(length_ > 1000) {
		float div = ceil((float)length_ / 1000.);
		chunk_dim[0] = (int)ceil(length_ / div);
		std::cerr << "! Changing chunk length to " << chunk_dim[0] << " stars." << std::endl;
	}
	H5::DataSpace dspace(rank, &(dim[0]));
	H5::DSetCreatPropList plist;
	plist.setDeflate(9);	// gzip compression level
	plist.setChunk(rank, &(chunk_dim[0]));
	float fillvalue = 0;
	plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
	
	H5::DataSet* dataset = new H5::DataSet(h5group->createDataSet(img, H5::PredType::NATIVE_FLOAT, dspace, plist));
	dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	/*
	 *  Attributes
	 */
	
	hsize_t att_dim = 2;
	H5::DataSpace att_dspace(1, &att_dim);
	
	H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
	H5::Attribute att_N = dataset->createAttribute("nPix", att_dtype, att_dspace);
	att_N.write(att_dtype, &(rect_.N_bins));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
	att_min.write(att_dtype, &(rect_.min));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
	att_max.write(att_dtype, &(rect_.max));
	
	delete dataset;
	delete h5group;
	delete h5file;
}
/* only for Eigen3 matrix/vector */
void HDF5IO::saveVector(const std::string& GroupName,
  const std::string& Name, const RealVectorType& V)
{
    hsize_t Dim[1] = {hsize_t(V.size())};
    H5::DataSpace dspace(1,Dim);
    H5::Group FG = getGroup( GroupName );
    try{
      H5::Exception::dontPrint();
      H5::DataSet DataSet = FG.openDataSet(Name.c_str());
      DataSet.write(V.data(),H5::PredType::NATIVE_DOUBLE, dspace);
    } catch ( const H5::GroupIException not_found_error ){
      H5::DataSet DataSet = FG.createDataSet(Name.c_str(),
        H5::PredType::NATIVE_DOUBLE,dspace);
      DataSet.write(V.data(),H5::PredType::NATIVE_DOUBLE);
    }
    FG.close();
}
示例#4
0
    static void save_molecular_type(const MolecularTypeBase* mtb,
            std::vector<std::pair<ParticleID, Voxel> > voxels, H5::Group* group)
    {
        const Species species(mtb->species());
        boost::scoped_ptr<H5::Group> mtgroup(
                new H5::Group(group->createGroup(species.serial().c_str())));

        h5_species_struct property;
        property.radius = mtb->radius();
        property.D = mtb->D();
        const MolecularTypeBase* loc(mtb->location());
        if (loc->is_vacant())
            std::strcpy(property.location, "");
        else
            std::strcpy(property.location, loc->species().serial().c_str());
        property.is_structure = mtb->is_structure() ? 1 : 0;
        property.dimension = mtb->get_dimension();

        H5::CompType property_comp_type(get_property_comp());
        mtgroup->createAttribute("property", property_comp_type,
                H5::DataSpace(H5S_SCALAR)).write(property_comp_type, &property);

        // Save voxels
        const Integer num_voxels(voxels.size());
        std::size_t vidx(0);
        boost::scoped_array<h5_voxel_struct> h5_voxel_array(new h5_voxel_struct[num_voxels]);
        for (std::vector<std::pair<ParticleID, Voxel> >::const_iterator itr(voxels.begin());
                itr != voxels.end(); ++itr)
        {
            h5_voxel_array[vidx].lot = (*itr).first.lot();
            h5_voxel_array[vidx].serial = (*itr).first.serial();
            h5_voxel_array[vidx].coordinate = (*itr).second.coordinate();
            ++vidx;
        }

        H5::CompType voxel_comp_type(get_voxel_comp());
        hsize_t dims[] = {num_voxels};
        H5::DataSpace dspace(/* RANK= */1, dims);
        boost::scoped_ptr<H5::DataSet> dset(new H5::DataSet(
            mtgroup->createDataSet("voxels", voxel_comp_type, dspace)));
        dset->write(h5_voxel_array.get(), dset->getDataType());
    }
void HDF5IO::saveStdVector(const std::string& GroupName, const std::string& Name,
    const std::vector<size_t>& V)
{
  try{
    hsize_t Dim[1] = {hsize_t(V.size())};
    H5::DataSpace dspace(1,Dim);
    H5::Group FG = getGroup( GroupName );
    try{
      H5::Exception::dontPrint();
      H5::DataSet DataSet = FG.openDataSet(Name.c_str());
      DataSet.write(V.data(),H5::PredType::NATIVE_ULONG, dspace);
    } catch ( const H5::GroupIException not_found_error ){
      H5::DataSet DataSet = FG.createDataSet(Name.c_str(),
        H5::PredType::NATIVE_ULONG, dspace);
      DataSet.write(V.data(), H5::PredType::NATIVE_ULONG);
    }
    FG.close();
  } catch( const H5::Exception err ){
    RUNTIME_ERROR("HDF5IO::saveUlongStdVector ");
  }
}
void HDF5IO::saveVector(const std::string& GroupName,
  const std::string& Name, const ComplexVectorType& V)
{
  try{
    H5::CompType ComplexDataType = this->openCompType("complex");
    hsize_t Dim[1] = {hsize_t(V.size())};
    H5::DataSpace dspace(1,Dim);
    H5::Group FG = getGroup( GroupName );
    try{
      H5::Exception::dontPrint();
      H5::DataSet DataSet = FG.openDataSet(Name.c_str());
      DataSet.write(V.data(), ComplexDataType, dspace);
    } catch ( const H5::GroupIException not_found_error ){
      H5::DataSet DataSet = FG.createDataSet(Name.c_str(), ComplexDataType,
        dspace);
      DataSet.write(V.data(), ComplexDataType);
    }
    FG.close();
  } catch ( const H5::DataSetIException error ){
    error.printError();
    RUNTIME_ERROR("HDF5IO::saveComplexVector at ");
  }
}
示例#7
0
void TChainWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& chain, const std::string& meta) {
	H5::H5File* h5file = H5Utils::openFile(fname);
	H5::Group* h5group = H5Utils::openGroup(h5file, group);
	
	// Dataset properties: optimized for reading/writing entire buffer at once
	int rank = 3;
	hsize_t dim[3] = {length_, nSamples_+2, nDim_};
	H5::DataSpace dspace(rank, &(dim[0]));
	H5::DSetCreatPropList plist;
	plist.setDeflate(9);	// gzip compression level
	plist.setChunk(rank, &(dim[0]));
	float fillvalue = 0;
	plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
	
	H5::DataSet* dataset = NULL;
	try {
		dataset = new H5::DataSet(h5group->createDataSet(chain, H5::PredType::NATIVE_FLOAT, dspace, plist));
	} catch(H5::GroupIException &group_exception) {
		std::cerr << "Could not create dataset for chain." << std::endl;
		std::cerr << "Dataset '" << group << "/" << chain << "' most likely already exists." << std::endl;
		throw;
	}
	
	dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	if(meta == "") {	// Store metadata as attributes
		bool *converged = new bool[length_];
		float *lnZ = new float[length_];
		for(unsigned int i=0; i<length_; i++) {
			converged[i] = metadata[i].converged;
			lnZ[i] = metadata[i].lnZ;
		}
		
		// Allow large attributes to be stored in dense storage, versus compact (which has 64 kB limit)
		//if(length_ > 5) {
		//	hid_t dID = dataset->getCreatePlist().getId();
		//	herr_t res = H5Pset_attr_phase_change(dID, 0, 0);
		//	std::cerr << res << std::endl;
		//	if(res < 0) {
		//		std::cerr << "Failed to specify dense storage." << std::endl;
		//	}
		//}
		
		H5::DataSpace convSpace(1, &(dim[0]));
		H5::Attribute convAtt = dataset->createAttribute("converged", H5::PredType::NATIVE_CHAR, convSpace);
		convAtt.write(H5::PredType::NATIVE_CHAR, reinterpret_cast<char*>(converged));
		
		H5::DataSpace lnZSpace(1, &(dim[0]));
		H5::Attribute lnZAtt = dataset->createAttribute("ln(Z)", H5::PredType::NATIVE_FLOAT, lnZSpace);
		lnZAtt.write(H5::PredType::NATIVE_FLOAT, lnZ);
		
		delete[] converged;
		delete[] lnZ;
	} else {	 	// Store metadata as separate dataset
		H5::CompType metaType(sizeof(TChainMetadata));
		metaType.insertMember("converged", HOFFSET(TChainMetadata, converged), H5::PredType::NATIVE_CHAR);
		metaType.insertMember("ln(Z)", HOFFSET(TChainMetadata, lnZ), H5::PredType::NATIVE_FLOAT);
		
		rank = 1;
		H5::DataSpace metaSpace(rank, &(dim[0]));
		H5::DSetCreatPropList metaProp;
		TChainMetadata emptyMetadata = {0, 0};
		metaProp.setFillValue(metaType, &emptyMetadata);
		metaProp.setDeflate(9);
		metaProp.setChunk(rank, &(dim[0]));
		
		H5::DataSet* metaDataset = new H5::DataSet(h5group->createDataSet(meta, metaType, metaSpace, metaProp));
		metaDataset->write(metadata.data(), metaType);
		
		delete metaDataset;
		metaDataset = NULL;
	}
	
	delete dataset;
	delete h5group;
	delete h5file;
	
	//std::cerr << "Cleaned up." << std::endl;
}
示例#8
0
bool save_mat_image(cv::Mat& img, TRect& rect, std::string fname, std::string group_name,
                    std::string dset_name, std::string dim1, std::string dim2, int compression) {
	assert((img.dims == 2) && (img.rows == rect.N_bins[0]) && (img.cols == rect.N_bins[1]));
	
	if((compression<0) || (compression > 9)) {
		std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
		return false;
	}
	
	H5::Exception::dontPrint();
	
	H5::H5File *file = H5Utils::openFile(fname);
	if(file == NULL) { return false; }
	
	H5::Group *group = H5Utils::openGroup(file, group_name);
	if(group == NULL) {
		delete file;
		return false;
	}
	
	/*
	 *  Image Data
	 */
	
	// Creation property list
	H5::DSetCreatPropList plist;
	int rank = 2;
	hsize_t dim[2] = {rect.N_bins[0], rect.N_bins[1]};
	plist.setDeflate(compression);	// gzip compression level
	float fillvalue = 0;
	plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
	plist.setChunk(rank, &(dim[0]));
	H5::DataSpace dspace(rank, &(dim[0]));
	
	H5::DataSet* dataset;
	try {
		dataset = new H5::DataSet(group->createDataSet(dset_name, H5::PredType::NATIVE_FLOAT, dspace, plist));
	} catch(H5::FileIException create_dset_err) {
		std::cerr << "Unable to create dataset '" << dset_name << "'." << std::endl;
		delete group;
		delete file;
		return false;
	}
	
	float *buf = new float[rect.N_bins[0]*rect.N_bins[1]];
	for(size_t j=0; j<rect.N_bins[0]; j++) {
		for(size_t k=0; k<rect.N_bins[1]; k++) {
			buf[rect.N_bins[1]*j + k] = img.at<double>(j,k);
			/*float tmp = img.at<double>(j,k);
			if(tmp > 0.) {
				std::cerr << j << ", " << k << " --> " << j + rect.N_bins[0]*k << " --> " << tmp << std::endl;
			}*/
		}
	}
	dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	/*
	 *  Attributes
	 */
	
	hsize_t att_dim = 2;
	H5::DataSpace att_dspace(1, &att_dim);
	
	H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
	H5::Attribute att_N = dataset->createAttribute("N_pix", att_dtype, att_dspace);
	att_N.write(att_dtype, &(rect.N_bins));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
	att_min.write(att_dtype, &(rect.min));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
	att_max.write(att_dtype, &(rect.max));
	
	att_dim = 1;
	H5::StrType vls_type(0, H5T_VARIABLE);
	H5::DataSpace att_space_str(H5S_SCALAR);
	H5::Attribute att_name_1 = dataset->createAttribute("dim_name_1", vls_type, att_space_str);
	att_name_1.write(vls_type, dim1);
	H5::Attribute att_name_2 = dataset->createAttribute("dim_name_2",  vls_type, att_space_str);
	att_name_2.write(vls_type, dim2);
	
	file->close();
	
	delete[] buf;
	delete dataset;
	delete group;
	delete file;
	
	return true;
	
}
示例#9
0
bool TStellarData::save(const std::string& fname, const std::string& group, const std::string &dset, int compression) {
	if((compression < 0) || (compression > 9)) {
		std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
		return false;
	}
	
	hsize_t nstars = star.size();
	if(nstars == 0) {
		std::cerr << "! No stars to write." << std::endl;
		return false;
	}
	
	H5::Exception::dontPrint();
	
	H5::H5File *file = H5Utils::openFile(fname);
	if(file == NULL) { return false; }
	
	H5::Group *gp = H5Utils::openGroup(file, group);
	if(gp == NULL) {
		delete file;
		return false;
	}
	
	/*
	 *  Photometry
	 */
	
	// Datatype
	hsize_t nbands = NBANDS;
	H5::ArrayType f4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
	H5::ArrayType u4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
	H5::CompType dtype(sizeof(TFileData));
	dtype.insertMember("obj_id", HOFFSET(TFileData, obj_id), H5::PredType::NATIVE_UINT64);
	dtype.insertMember("l", HOFFSET(TFileData, l), H5::PredType::NATIVE_DOUBLE);
	dtype.insertMember("b", HOFFSET(TFileData, b), H5::PredType::NATIVE_DOUBLE);
	dtype.insertMember("mag", HOFFSET(TFileData, mag), f4arr);
	dtype.insertMember("err", HOFFSET(TFileData, err), f4arr);
	dtype.insertMember("maglimit", HOFFSET(TFileData, maglimit), f4arr);
	dtype.insertMember("nDet", HOFFSET(TFileData, N_det), u4arr);
	dtype.insertMember("EBV", HOFFSET(TFileData, EBV), H5::PredType::NATIVE_FLOAT);
	
	// Dataspace
	hsize_t dim = nstars;
	H5::DataSpace dspace(1, &dim);
	
	// Property List
	H5::DSetCreatPropList plist;
	plist.setChunk(1, &nstars);
	plist.setDeflate(compression);
	
	// Dataset
	H5::DataSet dataset = gp->createDataSet(dset, dtype, dspace, plist);
	
	// Write dataset
	TFileData* data = new TFileData[nstars];
	for(size_t i=0; i<nstars; i++) {
		data[i].obj_id = star[i].obj_id;
		data[i].l = star[i].l;
		data[i].b = star[i].b;
		for(size_t k=0; k<NBANDS; k++) {
			data[i].mag[k] = star[i].m[k];
			data[i].err[k] = star[i].err[k];
			data[i].maglimit[k] = star[i].maglimit[k];
		}
		data[i].EBV = star[i].EBV;
	}
	dataset.write(data, dtype);
	
	/*
	 *  Attributes
	 */
	
	dim = 1;
	H5::DataSpace att_dspace(1, &dim);
	
	H5::PredType att_dtype = H5::PredType::NATIVE_UINT64;
	H5::Attribute att_healpix_index = dataset.createAttribute("healpix_index", att_dtype, att_dspace);
	att_healpix_index.write(att_dtype, &healpix_index);
	
	att_dtype = H5::PredType::NATIVE_UINT32;
	H5::Attribute att_nside = dataset.createAttribute("nside", att_dtype, att_dspace);
	att_nside.write(att_dtype, &nside);
	
	att_dtype = H5::PredType::NATIVE_UCHAR;
	H5::Attribute att_nested = dataset.createAttribute("nested", att_dtype, att_dspace);
	att_nested.write(att_dtype, &nested);
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_l = dataset.createAttribute("l", att_dtype, att_dspace);
	att_l.write(att_dtype, &l);
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_b = dataset.createAttribute("b", att_dtype, att_dspace);
	att_b.write(att_dtype, &b);
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_EBV = dataset.createAttribute("EBV", att_dtype, att_dspace);
	att_EBV.write(att_dtype, &EBV);
	
	file->close();
	
	delete[] data;
	delete gp;
	delete file;
	
	return true;
}