Ejemplo n.º 1
0
H5::DSetCreatPropList hdfutil::CreatePropList () {
	H5::DSetCreatPropList plist;
	hid_t dset_cplist = plist.getId();
	// disable time-stamping of datasets
	/*herr_t ret_val =*/ H5Pset_obj_track_times(dset_cplist, false);
    return plist;
}
Ejemplo n.º 2
0
DataSet DataSet::create(const H5::CommonFG &parent,
                        const std::string &name,
                        const H5::DataType &fileType,
                        const NDSize &size,
                        const NDSize &maxsize,
                        const NDSize &chunks,
                        bool max_size_unlimited,
                        bool guess_chunks)
{
    H5::DataSpace space;

    if (size) {
        if (maxsize) {
            space = DataSpace::create(size, maxsize);
        } else {
            space = DataSpace::create(size, max_size_unlimited);
        }
    }

    H5::DSetCreatPropList plcreate = H5::DSetCreatPropList::DEFAULT;

    if (chunks) {
        int rank = static_cast<int>(chunks.size());
        plcreate.setChunk(rank, chunks.data());
    } else if (guess_chunks) {
        NDSize guessedChunks = DataSet::guessChunking(size, fileType.getSize());
        plcreate.setChunk(static_cast<int>(guessedChunks.size()), guessedChunks.data());
    }

    H5::DataSet dset = parent.createDataSet(name, fileType, space);
    return DataSet(dset);
}
Ejemplo n.º 3
0
void TImgWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& img) {
	H5::H5File* h5file = H5Utils::openFile(fname);
	H5::Group* h5group = H5Utils::openGroup(h5file, group);
	
	// Dataset properties: optimized for reading/writing entire buffer at once
	int rank = 3;
	hsize_t dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
	hsize_t chunk_dim[3] = {length_, rect_.N_bins[0], rect_.N_bins[1]};
	if(length_ > 1000) {
		float div = ceil((float)length_ / 1000.);
		chunk_dim[0] = (int)ceil(length_ / div);
		std::cerr << "! Changing chunk length to " << chunk_dim[0] << " stars." << std::endl;
	}
	H5::DataSpace dspace(rank, &(dim[0]));
	H5::DSetCreatPropList plist;
	plist.setDeflate(9);	// gzip compression level
	plist.setChunk(rank, &(chunk_dim[0]));
	float fillvalue = 0;
	plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
	
	H5::DataSet* dataset = new H5::DataSet(h5group->createDataSet(img, H5::PredType::NATIVE_FLOAT, dspace, plist));
	dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	/*
	 *  Attributes
	 */
	
	hsize_t att_dim = 2;
	H5::DataSpace att_dspace(1, &att_dim);
	
	H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
	H5::Attribute att_N = dataset->createAttribute("nPix", att_dtype, att_dspace);
	att_N.write(att_dtype, &(rect_.N_bins));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
	att_min.write(att_dtype, &(rect_.min));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
	att_max.write(att_dtype, &(rect_.max));
	
	delete dataset;
	delete h5group;
	delete h5file;
}
Ejemplo n.º 4
0
void File::create( uint64_t N, bool bDouble ) {
    hsize_t chunkSize[2],dataSize[2], maxSize[2];

    maxSize[0] = H5S_UNLIMITED;
    chunkSize[0] = 16;
    dataSize[0] = m_classes.size();
    H5::DataSpace spaceDisk(1,dataSize,maxSize);

    H5::DSetCreatPropList propDisk;
    propDisk.setChunk(1,chunkSize);
    propDisk.setFletcher32();

    H5::Group darkGroup(createGroup( "dark" ));
    H5::DataSet classesDataSet(
	darkGroup.createDataSet("classes", m_classType,
				spaceDisk,propDisk) );

    H5::Group parmGroup(createGroup( "parameters" ));

    maxSize[0] = H5S_UNLIMITED;
    maxSize[1] = 3;
    chunkSize[0] = CHUNK_SIZE;
    chunkSize[1] = 1;
    dataSize[0] = N;
    dataSize[1] = 3;

    H5::DataSpace spaceDisk2(2,dataSize,maxSize);

    H5::DSetCreatPropList propDisk2;
    //propDisk.setLayout(H5D_CHUNKED);
    propDisk2.setChunk(2,chunkSize);
    propDisk2.setFletcher32();

    H5::DataSet positionDataSet(
	darkGroup.createDataSet(
	    "position",
	    bDouble ? H5::PredType::NATIVE_DOUBLE : H5::PredType::NATIVE_FLOAT,
	    spaceDisk2,propDisk2) );

    H5::DataSet velocityDataSet(
	darkGroup.createDataSet(
	    "velocity",
	    bDouble ? H5::PredType::NATIVE_DOUBLE : H5::PredType::NATIVE_FLOAT,
	    spaceDisk2,propDisk2) );

    }
void BufferedHDF2DArray<T>::Create(H5::CommonFG *_container, string _datasetName, unsigned int _rowLength) {
    container   = _container;
    datasetName = _datasetName;
    rowLength   = (unsigned int)_rowLength;
    //
    // Make life easy if the buffer is too small to fit a row --
    // resize it so that rows may be copied and written out in an
    // atomic unit.
    //
    if (this->bufferSize < rowLength) {
        // When the buffer size is greater than 0, the write buffer
        // should exist.
        if (this->bufferSize > 0) {
            assert(this->writeBuffer != NULL);
            delete[] this->writeBuffer;
        }
        this->writeBuffer = new T[rowLength];
        this->bufferSize = rowLength;
    }

    hsize_t dataSize[2]    = {0, hsize_t(rowLength)};
    hsize_t maxDataSize[2] = {H5S_UNLIMITED, hsize_t(rowLength)};
    H5::DataSpace fileSpace(2, dataSize, maxDataSize);
    H5::DSetCreatPropList cparms;

    /*
     * For some reason, chunking must be enabled when creating a dataset
     * that  has an unlimited dimension.  Of course, this is not
     * mentioned in the hdf5 c++ documentation, because that
     * docuemntation was written for people who enjoy learning how to
     * use an API by reading comments in source code.
     */
    hsize_t chunkDims[2] = {16384, hsize_t(rowLength)};
    cparms.setChunk( 2, chunkDims );
    TypedCreate(fileSpace, cparms);
    fileSpace.close();

    //
    // Set some flags that indicate this dataset is ready for writing.
    //
    fileDataSpaceInitialized = true;
    isInitialized = true;
}
Ejemplo n.º 6
0
int main()
{
  std::string fname = "testhdf5file.h5";
  H5::H5File f( fname, H5F_ACC_TRUNC );
  std::string dsetname = "dset1";

  const hsize_t nranks = 2;
  const hsize_t ncols = 20;
  const hsize_t nrows = 0; //start with zero rows.
      
  hsize_t  dims[nranks] = {nrows, ncols};
  hsize_t  max_dims[nranks] = {H5S_UNLIMITED, ncols};
  
  H5::DataSpace dataspace( nranks, dims, max_dims );
  H5::DSetCreatPropList prop; //could set properties, but whatever.
  
  const hsize_t nrows_chunk = 100; //Need to mess with CACHE size too!
  hsize_t  chunk_dims[ndims] = { nrows_chunk, ncols};
  prop.setChunk(ndims, chunk_dims);


  //Create the dataset 
  H5::DataSet ds = f.createDataSet( dsetname, H5::PredType::NATIVE_DOUBLE,
				    dataspace, prop);
  

  std::vector<double> rowtowrite( ncols, 5.0 );
  
  for(size_t t=0; t<10000000; ++t)
    {
      writerow( ds, rowtowrite );

      std::vector<double> lastrow( ncols, 0.0 );
      lastrow = readlastrow( ds );
      if( lastrow[0] != 5.0 )
	{
	  fprintf(stderr, "ERROR IN READING...\n");
	  exit(1);
	}
    }
  
  return 0;
}
Ejemplo n.º 7
0
void TChainWriteBuffer::write(const std::string& fname, const std::string& group, const std::string& chain, const std::string& meta) {
	H5::H5File* h5file = H5Utils::openFile(fname);
	H5::Group* h5group = H5Utils::openGroup(h5file, group);
	
	// Dataset properties: optimized for reading/writing entire buffer at once
	int rank = 3;
	hsize_t dim[3] = {length_, nSamples_+2, nDim_};
	H5::DataSpace dspace(rank, &(dim[0]));
	H5::DSetCreatPropList plist;
	plist.setDeflate(9);	// gzip compression level
	plist.setChunk(rank, &(dim[0]));
	float fillvalue = 0;
	plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
	
	H5::DataSet* dataset = NULL;
	try {
		dataset = new H5::DataSet(h5group->createDataSet(chain, H5::PredType::NATIVE_FLOAT, dspace, plist));
	} catch(H5::GroupIException &group_exception) {
		std::cerr << "Could not create dataset for chain." << std::endl;
		std::cerr << "Dataset '" << group << "/" << chain << "' most likely already exists." << std::endl;
		throw;
	}
	
	dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	if(meta == "") {	// Store metadata as attributes
		bool *converged = new bool[length_];
		float *lnZ = new float[length_];
		for(unsigned int i=0; i<length_; i++) {
			converged[i] = metadata[i].converged;
			lnZ[i] = metadata[i].lnZ;
		}
		
		// Allow large attributes to be stored in dense storage, versus compact (which has 64 kB limit)
		//if(length_ > 5) {
		//	hid_t dID = dataset->getCreatePlist().getId();
		//	herr_t res = H5Pset_attr_phase_change(dID, 0, 0);
		//	std::cerr << res << std::endl;
		//	if(res < 0) {
		//		std::cerr << "Failed to specify dense storage." << std::endl;
		//	}
		//}
		
		H5::DataSpace convSpace(1, &(dim[0]));
		H5::Attribute convAtt = dataset->createAttribute("converged", H5::PredType::NATIVE_CHAR, convSpace);
		convAtt.write(H5::PredType::NATIVE_CHAR, reinterpret_cast<char*>(converged));
		
		H5::DataSpace lnZSpace(1, &(dim[0]));
		H5::Attribute lnZAtt = dataset->createAttribute("ln(Z)", H5::PredType::NATIVE_FLOAT, lnZSpace);
		lnZAtt.write(H5::PredType::NATIVE_FLOAT, lnZ);
		
		delete[] converged;
		delete[] lnZ;
	} else {	 	// Store metadata as separate dataset
		H5::CompType metaType(sizeof(TChainMetadata));
		metaType.insertMember("converged", HOFFSET(TChainMetadata, converged), H5::PredType::NATIVE_CHAR);
		metaType.insertMember("ln(Z)", HOFFSET(TChainMetadata, lnZ), H5::PredType::NATIVE_FLOAT);
		
		rank = 1;
		H5::DataSpace metaSpace(rank, &(dim[0]));
		H5::DSetCreatPropList metaProp;
		TChainMetadata emptyMetadata = {0, 0};
		metaProp.setFillValue(metaType, &emptyMetadata);
		metaProp.setDeflate(9);
		metaProp.setChunk(rank, &(dim[0]));
		
		H5::DataSet* metaDataset = new H5::DataSet(h5group->createDataSet(meta, metaType, metaSpace, metaProp));
		metaDataset->write(metadata.data(), metaType);
		
		delete metaDataset;
		metaDataset = NULL;
	}
	
	delete dataset;
	delete h5group;
	delete h5file;
	
	//std::cerr << "Cleaned up." << std::endl;
}
Ejemplo n.º 8
0
bool TChain::save(std::string fname, std::string group_name, size_t index,
                  std::string dim_name, int compression, int subsample,
                  bool converged, float lnZ) const {
	if((compression<0) || (compression > 9)) {
		std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
		return false;
	}
	
	H5::Exception::dontPrint();
	
	H5::H5File *file = H5Utils::openFile(fname);
	if(file == NULL) { return false; }
	
	/*
	try {
		file->unlink(group_name);
	} catch(...) {
		// pass
	}
	*/
	
	H5::Group *group = H5Utils::openGroup(file, group_name);
	if(group == NULL) {
		delete file;
		return false;
	}
	
	/*
	 *  Attributes
	 */
	
	// Datatype
	H5::CompType att_type(sizeof(TChainAttribute));
	hid_t tid = H5Tcopy(H5T_C_S1);
	H5Tset_size(tid, H5T_VARIABLE);
	att_type.insertMember("dim_name", HOFFSET(TChainAttribute, dim_name), tid);
	//att_type.insertMember("total_weight", HOFFSET(TChainAttribute, total_weight), H5::PredType::NATIVE_FLOAT);
	//att_type.insertMember("ndim", HOFFSET(TChainAttribute, ndim), H5::PredType::NATIVE_UINT64);
	//att_type.insertMember("length", HOFFSET(TChainAttribute, length), H5::PredType::NATIVE_UINT64);
	
	// Dataspace
	int att_rank = 1;
	hsize_t att_dim = 1;
	H5::DataSpace att_space(att_rank, &att_dim);
	
	// Dataset
	//H5::Attribute att = group->createAttribute("parameter names", att_type, att_space);
	
	TChainAttribute att_data;
	att_data.dim_name = new char[dim_name.size()+1];
	std::strcpy(att_data.dim_name, dim_name.c_str());
	//att_data.total_weight = total_weight;
	//att_data.ndim = N;
	//att_data.length = length;
	
	//att.write(att_type, &att_data);
	delete[] att_data.dim_name;
	
	//int att_rank = 1;
	//hsize_t att_dim = 1;
	
	H5::DataType conv_dtype = H5::PredType::NATIVE_UCHAR;
	H5::DataSpace conv_dspace(att_rank, &att_dim);
	//H5::Attribute conv_att = H5Utils::openAttribute(group, "converged", conv_dtype, conv_dspace);
	//conv_att.write(conv_dtype, &converged);
	
	H5::DataType lnZ_dtype = H5::PredType::NATIVE_FLOAT;
	H5::DataSpace lnZ_dspace(att_rank, &att_dim);
	//H5::Attribute lnZ_att = H5Utils::openAttribute(group, "ln Z", lnZ_dtype, lnZ_dspace);
	//lnZ_att.write(lnZ_dtype, &lnZ);
	
	// Creation property list to be used for all three datasets
	H5::DSetCreatPropList plist;
	//plist.setDeflate(compression);	// gzip compression level
	float fillvalue = 0;
	plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
	
	H5D_layout_t layout = H5D_COMPACT;
	plist.setLayout(layout);
	
	/*
	 *  Choose subsample of points in chain
	 */
	size_t *el_idx = NULL;
	size_t *subsample_idx = NULL;
	if(subsample > 0) {
		size_t tot_weight_tmp = (size_t)ceil(total_weight);
		el_idx = new size_t[tot_weight_tmp];
		size_t unrolled_idx = 0;
		size_t chain_idx = 0;
		std::vector<double>::const_iterator it, it_end;
		it_end = w.end();
		for(it = w.begin(); it != it_end; ++it, chain_idx++) {
			for(size_t n = unrolled_idx; n < unrolled_idx + (size_t)(*it); n++) {
				el_idx[n] = chain_idx;
			}
			unrolled_idx += (size_t)(*it);
		}
		
		assert(chain_idx == length);
		
		gsl_rng *r;
		seed_gsl_rng(&r);
		subsample_idx = new size_t[tot_weight_tmp];
		for(size_t i=0; i<subsample; i++) {
			subsample_idx[i] = el_idx[gsl_rng_uniform_int(r, tot_weight_tmp)];
		}
	}
	
	
	/*
	 *  Coordinates
	 */
	
	// Dataspace
	hsize_t dim;
	if(subsample > 0) {
		dim = subsample;
	} else {
		dim = length;
	}
	// Chunking (required for compression)
	int rank = 2;
	hsize_t coord_dim[2] = {dim, N};
	//if(dim < chunk) {
	//plist.setChunk(rank, &(coord_dim[0]));
	//} else {
	//	plist.setChunk(rank, &chunk);
	//}
	H5::DataSpace x_dspace(rank, &(coord_dim[0]));
	
	// Dataset
	//std::stringstream x_dset_path;
	//x_dset_path << group_name << "/chain/coords";
	std::stringstream coordname;
	coordname << "coords " << index;
	H5::DataSet* x_dataset = new H5::DataSet(group->createDataSet(coordname.str(), H5::PredType::NATIVE_FLOAT, x_dspace, plist));
	
	// Write
	float *buf = new float[N*dim];
	if(subsample > 0) {
		size_t tmp_idx;
		for(size_t i=0; i<subsample; i++) {
			tmp_idx = subsample_idx[i];
			for(size_t k=0; k<N; k++) {
				buf[N*i+k] = x[N*tmp_idx+k];
			}
		}
	} else {
		for(size_t i=0; i<dim; i++) { buf[i] = x[i]; }
	}
	x_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	
	/*
	 *  Weights
	 */
	
	// Dataspace
	if(subsample <= 0) {
		dim = w.size();
		
		rank = 1;
		H5::DataSpace w_dspace(rank, &dim);
		
		// Dataset
		//std::stringstream w_dset_path;
		//w_dset_path << group_name << "/chain/weights";
		H5::DataSet* w_dataset = new H5::DataSet(group->createDataSet("weights", H5::PredType::NATIVE_FLOAT, w_dspace, plist));
		
		// Write
		if(subsample > 0) {
			for(size_t i=0; i<subsample; i++) { buf[i] = 1.; }
		} else {
			assert(w.size() < x.size());
			for(size_t i=0; i<w.size(); i++) { buf[i] = w[i]; }
		}
		w_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
		
		delete w_dataset;
	}
	
	/*
	 *  Probability densities
	 */
	
	// Dataspace
	rank = 1;
	H5::DataSpace L_dspace(rank, &dim);
	
	// Dataset
	//std::stringstream L_dset_path;
	//L_dset_path << group_name << "/chain/probs";
	std::stringstream lnpname;
	lnpname << "ln_p " << index;
	H5::DataSet* L_dataset = new H5::DataSet(group->createDataSet(lnpname.str(), H5::PredType::NATIVE_FLOAT, L_dspace, plist));
	
	// Write
	if(subsample > 0) {
		for(size_t i=0; i<subsample; i++) { buf[i] = L[subsample_idx[i]]; }
	} else {
		assert(L.size() < x.size());
		for(size_t i=0; i<L.size(); i++) { buf[i] = L[i]; }
	}
	L_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	
	if(subsample > 0) {
		delete[] el_idx;
		delete[] subsample_idx;
	}
	
	delete[] buf;
	
	delete x_dataset;
	delete L_dataset;
	
	delete group;
	delete file;
	
	return true;
}
Ejemplo n.º 9
0
bool save_mat_image(cv::Mat& img, TRect& rect, std::string fname, std::string group_name,
                    std::string dset_name, std::string dim1, std::string dim2, int compression) {
	assert((img.dims == 2) && (img.rows == rect.N_bins[0]) && (img.cols == rect.N_bins[1]));
	
	if((compression<0) || (compression > 9)) {
		std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
		return false;
	}
	
	H5::Exception::dontPrint();
	
	H5::H5File *file = H5Utils::openFile(fname);
	if(file == NULL) { return false; }
	
	H5::Group *group = H5Utils::openGroup(file, group_name);
	if(group == NULL) {
		delete file;
		return false;
	}
	
	/*
	 *  Image Data
	 */
	
	// Creation property list
	H5::DSetCreatPropList plist;
	int rank = 2;
	hsize_t dim[2] = {rect.N_bins[0], rect.N_bins[1]};
	plist.setDeflate(compression);	// gzip compression level
	float fillvalue = 0;
	plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
	plist.setChunk(rank, &(dim[0]));
	H5::DataSpace dspace(rank, &(dim[0]));
	
	H5::DataSet* dataset;
	try {
		dataset = new H5::DataSet(group->createDataSet(dset_name, H5::PredType::NATIVE_FLOAT, dspace, plist));
	} catch(H5::FileIException create_dset_err) {
		std::cerr << "Unable to create dataset '" << dset_name << "'." << std::endl;
		delete group;
		delete file;
		return false;
	}
	
	float *buf = new float[rect.N_bins[0]*rect.N_bins[1]];
	for(size_t j=0; j<rect.N_bins[0]; j++) {
		for(size_t k=0; k<rect.N_bins[1]; k++) {
			buf[rect.N_bins[1]*j + k] = img.at<double>(j,k);
			/*float tmp = img.at<double>(j,k);
			if(tmp > 0.) {
				std::cerr << j << ", " << k << " --> " << j + rect.N_bins[0]*k << " --> " << tmp << std::endl;
			}*/
		}
	}
	dataset->write(buf, H5::PredType::NATIVE_FLOAT);
	
	/*
	 *  Attributes
	 */
	
	hsize_t att_dim = 2;
	H5::DataSpace att_dspace(1, &att_dim);
	
	H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
	H5::Attribute att_N = dataset->createAttribute("N_pix", att_dtype, att_dspace);
	att_N.write(att_dtype, &(rect.N_bins));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
	att_min.write(att_dtype, &(rect.min));
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
	att_max.write(att_dtype, &(rect.max));
	
	att_dim = 1;
	H5::StrType vls_type(0, H5T_VARIABLE);
	H5::DataSpace att_space_str(H5S_SCALAR);
	H5::Attribute att_name_1 = dataset->createAttribute("dim_name_1", vls_type, att_space_str);
	att_name_1.write(vls_type, dim1);
	H5::Attribute att_name_2 = dataset->createAttribute("dim_name_2",  vls_type, att_space_str);
	att_name_2.write(vls_type, dim2);
	
	file->close();
	
	delete[] buf;
	delete dataset;
	delete group;
	delete file;
	
	return true;
	
}
Ejemplo n.º 10
0
bool TStellarData::save(const std::string& fname, const std::string& group, const std::string &dset, int compression) {
	if((compression < 0) || (compression > 9)) {
		std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
		return false;
	}
	
	hsize_t nstars = star.size();
	if(nstars == 0) {
		std::cerr << "! No stars to write." << std::endl;
		return false;
	}
	
	H5::Exception::dontPrint();
	
	H5::H5File *file = H5Utils::openFile(fname);
	if(file == NULL) { return false; }
	
	H5::Group *gp = H5Utils::openGroup(file, group);
	if(gp == NULL) {
		delete file;
		return false;
	}
	
	/*
	 *  Photometry
	 */
	
	// Datatype
	hsize_t nbands = NBANDS;
	H5::ArrayType f4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
	H5::ArrayType u4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
	H5::CompType dtype(sizeof(TFileData));
	dtype.insertMember("obj_id", HOFFSET(TFileData, obj_id), H5::PredType::NATIVE_UINT64);
	dtype.insertMember("l", HOFFSET(TFileData, l), H5::PredType::NATIVE_DOUBLE);
	dtype.insertMember("b", HOFFSET(TFileData, b), H5::PredType::NATIVE_DOUBLE);
	dtype.insertMember("mag", HOFFSET(TFileData, mag), f4arr);
	dtype.insertMember("err", HOFFSET(TFileData, err), f4arr);
	dtype.insertMember("maglimit", HOFFSET(TFileData, maglimit), f4arr);
	dtype.insertMember("nDet", HOFFSET(TFileData, N_det), u4arr);
	dtype.insertMember("EBV", HOFFSET(TFileData, EBV), H5::PredType::NATIVE_FLOAT);
	
	// Dataspace
	hsize_t dim = nstars;
	H5::DataSpace dspace(1, &dim);
	
	// Property List
	H5::DSetCreatPropList plist;
	plist.setChunk(1, &nstars);
	plist.setDeflate(compression);
	
	// Dataset
	H5::DataSet dataset = gp->createDataSet(dset, dtype, dspace, plist);
	
	// Write dataset
	TFileData* data = new TFileData[nstars];
	for(size_t i=0; i<nstars; i++) {
		data[i].obj_id = star[i].obj_id;
		data[i].l = star[i].l;
		data[i].b = star[i].b;
		for(size_t k=0; k<NBANDS; k++) {
			data[i].mag[k] = star[i].m[k];
			data[i].err[k] = star[i].err[k];
			data[i].maglimit[k] = star[i].maglimit[k];
		}
		data[i].EBV = star[i].EBV;
	}
	dataset.write(data, dtype);
	
	/*
	 *  Attributes
	 */
	
	dim = 1;
	H5::DataSpace att_dspace(1, &dim);
	
	H5::PredType att_dtype = H5::PredType::NATIVE_UINT64;
	H5::Attribute att_healpix_index = dataset.createAttribute("healpix_index", att_dtype, att_dspace);
	att_healpix_index.write(att_dtype, &healpix_index);
	
	att_dtype = H5::PredType::NATIVE_UINT32;
	H5::Attribute att_nside = dataset.createAttribute("nside", att_dtype, att_dspace);
	att_nside.write(att_dtype, &nside);
	
	att_dtype = H5::PredType::NATIVE_UCHAR;
	H5::Attribute att_nested = dataset.createAttribute("nested", att_dtype, att_dspace);
	att_nested.write(att_dtype, &nested);
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_l = dataset.createAttribute("l", att_dtype, att_dspace);
	att_l.write(att_dtype, &l);
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_b = dataset.createAttribute("b", att_dtype, att_dspace);
	att_b.write(att_dtype, &b);
	
	att_dtype = H5::PredType::NATIVE_DOUBLE;
	H5::Attribute att_EBV = dataset.createAttribute("EBV", att_dtype, att_dspace);
	att_EBV.write(att_dtype, &EBV);
	
	file->close();
	
	delete[] data;
	delete gp;
	delete file;
	
	return true;
}
Ejemplo n.º 11
0
// Bundle management
void Bundle2::save(const boost::filesystem::path& fileName) const {
	// Creating HDF5 file
	H5::H5File bundleFile(fileName.string(), H5F_ACC_TRUNC);
	storeParameters(bundleFile);

	H5::DataSpace scalar;

	// Saving POI
	H5::Group poiGroup = bundleFile.createGroup("/POI");

	H5::Attribute attr = poiGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
	hsize_t count = poi_.size();
	attr.write(H5::PredType::NATIVE_HSIZE, &count);
	attr.close();

	for(size_t frame = 0; frame < poi_.size(); ++frame) {
		const std::string frameGroupName = boost::str(boost::format("Frame %1$04d") % frame);
		H5::Group frameGroup = poiGroup.createGroup(frameGroupName);

		count = poi_[frame].size();
		attr = frameGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
		attr.write(H5::PredType::NATIVE_HSIZE, &count);
		attr.close();

		for(size_t camera = 0; camera < poi_[frame].size(); ++camera)
			poi_[frame][camera].save(frameGroup, camera);

		frameGroup.close();
	}

	poiGroup.close();

	// Saving key frames
	H5::Group bundleGroup = bundleFile.createGroup("/Bundle");

	H5::Group framesGroup = bundleGroup.createGroup("Frames");

	count = frames_.size();
	attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
	attr.write(H5::PredType::NATIVE_HSIZE, &count);
	attr.close();

	for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
		(*it)->save(framesGroup);
	}

	framesGroup.close();

	// Saving tracks
	const hsize_t chunkDim[] = { 2, 1 };
	H5::DSetCreatPropList propList;
	propList.setLayout(H5D_CHUNKED);
	propList.setChunk(2, chunkDim);
	propList.setDeflate(9);

	H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE);

	hsize_t tracksDim[] = { tracks_.size(), 2 };
    hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 };
	H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim);

	H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList);

	for(size_t i = 0; i < tracks_.size(); ++i)
		tracks_[i]->save(tracksDataset, i);

	tracksDataset.close();
	tracksDataspace.close();
	tracksDatasetType.close();
	propList.close();

	bundleGroup.close();

	scalar.close();
	bundleFile.close();
}
Ejemplo n.º 12
0
void Bundle2::initGeometryStream_() {
	// Creating group Geometry
	H5::Group geometryGroup = streamFile_->createGroup("/Geometry");

	// Saving poses
	const hsize_t posesChunkDim[] = { 3, 12 };
	H5::DSetCreatPropList posesPropList;
	posesPropList.setLayout(H5D_CHUNKED);
	posesPropList.setChunk(2, posesChunkDim);
	posesPropList.setDeflate(9);

	const hsize_t posesMaxDim[] = { H5S_UNLIMITED, 12 };
	const hsize_t posesCurDim[] = { frames_.size(), 12 };
	H5::DataSpace posesDS(2, posesCurDim, posesMaxDim);

	H5::DataSet posesDataSet = geometryGroup.createDataSet("Poses", H5::PredType::IEEE_F64LE, posesDS, posesPropList);

	double* posesData = (double*)malloc(frames_.size()*12*sizeof(double));
	size_t i = 0;
	for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
		posesData[i*12] = (*it)->pose()->t().x();
		posesData[i*12 + 1] = (*it)->pose()->t().y();
		posesData[i*12 + 2] = (*it)->pose()->t().z();

		core::Matrix<double> R = (*it)->pose()->R();
		posesData[i*12 + 3] = R[0][0];
		posesData[i*12 + 4] = R[1][0];
		posesData[i*12 + 5] = R[2][0];
		posesData[i*12 + 6] = R[0][1];
		posesData[i*12 + 7] = R[1][1];
		posesData[i*12 + 8] = R[2][1];
		posesData[i*12 + 9] = R[0][2];
		posesData[i*12 + 10] = R[1][2];
		posesData[i*12 + 11] = R[2][2];

		++i;
	}

	posesDataSet.write((const void*)posesData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL);
	free((void*)posesData);
	posesDataSet.close();
	posesDS.close();

	// Creating points dataset
	const hsize_t pointsChunkDim[] = {10, 3};
	H5::DSetCreatPropList pointsPropList;
	pointsPropList.setLayout(H5D_CHUNKED);
	pointsPropList.setChunk(2, pointsChunkDim);
	pointsPropList.setDeflate(9);

	const hsize_t pointsMaxDim[] = { H5S_UNLIMITED, 3 };
	const hsize_t pointsCurDim[] = { 0, 3 };
	H5::DataSpace pointsDS(2, pointsCurDim, pointsMaxDim);

	H5::DataSet pointsDataSet = geometryGroup.createDataSet("Points", H5::PredType::IEEE_F64LE, pointsDS, pointsPropList);

	pointsDataSet.close();
	pointsDS.close();

	// Creating inliers dataset
	const hsize_t inliersChunkDim[] = { 3 };
	H5::DSetCreatPropList inliersPropList;
	inliersPropList.setLayout(H5D_CHUNKED);
	inliersPropList.setChunk(1, inliersChunkDim);
	inliersPropList.setDeflate(9);

	const hsize_t inliersMaxDim[] = { H5S_UNLIMITED };
	const hsize_t inliersCurDim[] = { frames_.size() };
	H5::DataSpace inliersDS(1, inliersCurDim, inliersMaxDim);

	H5::VarLenType inliersType(&H5::PredType::STD_U8LE);

	H5::DataSet inliersDataSet = geometryGroup.createDataSet("Inliers", inliersType, inliersDS, inliersPropList);

	inliersDataSet.close();
	inliersType.close();
	inliersDS.close();

	// Creating curve dataset
	const hsize_t chunkDim[] = { 5 };
	H5::DSetCreatPropList propList;
	propList.setLayout(H5D_CHUNKED);
	propList.setChunk(1, chunkDim);
	propList.setDeflate(9);

	H5::VarLenType curveDatasetType(&H5::PredType::STD_U64LE);

	hsize_t curvesDim[] = { 0 };
	hsize_t curvesMaxDim[] = { H5S_UNLIMITED };
	H5::DataSpace curvesDataspace(1, curvesDim, curvesMaxDim);

	H5::DataSet curvesDataset = geometryGroup.createDataSet("Curves", curveDatasetType, curvesDataspace, propList);

	curvesDataset.close();
	curvesDataspace.close();
	curveDatasetType.close();
	propList.close();

	geometryGroup.close();
}
Ejemplo n.º 13
0
void Bundle2::initFrameStream_() {
	H5::DataSpace scalar;

	// Creating datasets for each frame
	H5::Group bundleGroup = streamFile_->createGroup("/Bundle");
	H5::Group framesGroup = bundleGroup.createGroup("Frames");

	hsize_t count = frames_.size();
	H5::Attribute attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
	attr.write(H5::PredType::NATIVE_HSIZE, &count);
	attr.close();

	// Defining frame dataset property
	hsize_t chunkDim[] = { 1, 2, 10 };
	H5::DSetCreatPropList propList;
	propList.setLayout(H5D_CHUNKED);
	propList.setChunk(3, chunkDim);
	propList.setDeflate(9);

	// Definig dataset dataspace
	hsize_t max_dim[] = { numCameras_, 2, H5S_UNLIMITED };
	hsize_t dim[] = { numCameras_, 2, 0 };
	H5::DataSpace ds(3, dim, max_dim);

	for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
		const std::string datasetName = boost::str(boost::format("Frame %1$04d") % (*it)->number());

		// Creating dataset
		H5::DataSet frameData = framesGroup.createDataSet(datasetName, H5::PredType::IEEE_F32LE, ds, propList);

		// Writing global number
		attr = frameData.createAttribute("globalNumber", H5::PredType::STD_U64LE, scalar);
		count = (*it)->globalNumber();
		attr.write(H5::PredType::NATIVE_HSIZE, &count);
		attr.close();

		// Clean up!
		frameData.close();
	}

	// Clean up!
	ds.close();
	propList.close();

	// Creating tracks dataset
	hsize_t chunkDim2[] = { 2, 10 };
	propList = H5::DSetCreatPropList();
	propList.setLayout(H5D_CHUNKED);
	propList.setChunk(2, chunkDim);
	propList.setDeflate(9);

	H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE);

	hsize_t tracksDim[] = { 0, 2 };
	hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 };
	H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim);

	H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList);

	tracksDataset.close();
	tracksDataspace.close();
	tracksDatasetType.close();
	propList.close();

	framesGroup.close();
	bundleGroup.close();
	scalar.close();
}
Ejemplo n.º 14
0
void Bundle2::saveGeometry(const boost::filesystem::path& fileName) const {
	H5::H5File bundleFile;
	bundleFile.openFile(fileName.string(), H5F_ACC_RDWR);

	H5::Group rootGroup = bundleFile.openGroup("/");

	// If the group "Geometry" exists, delete it!
	if(checkGeometry_(bundleFile)) {
		rootGroup.unlink("Geometry");
	}

	// Creating group Geometry
	H5::Group geometryGroup = rootGroup.createGroup("Geometry");

	// Saving poses
	const hsize_t posesChunkDim[] = { 3, 12 };
	H5::DSetCreatPropList posesPropList;
	posesPropList.setLayout(H5D_CHUNKED);
	posesPropList.setChunk(2, posesChunkDim);
	posesPropList.setDeflate(9);

	const hsize_t posesMaxDim[] = { H5S_UNLIMITED, 12 };
	const hsize_t posesCurDim[] = { frames_.size(), 12 };
	H5::DataSpace posesDS(2, posesCurDim, posesMaxDim);

	H5::DataSet posesDataSet = geometryGroup.createDataSet("Poses", H5::PredType::IEEE_F64LE, posesDS, posesPropList);

	double* posesData = (double*)malloc(frames_.size()*12*sizeof(double));
	size_t i = 0;
	for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
		posesData[i*12] = (*it)->pose()->t().x();
		posesData[i*12 + 1] = (*it)->pose()->t().y();
		posesData[i*12 + 2] = (*it)->pose()->t().z();

		core::Matrix<double> R = (*it)->pose()->R();
		posesData[i*12 + 3] = R[0][0];
		posesData[i*12 + 4] = R[1][0];
		posesData[i*12 + 5] = R[2][0];
		posesData[i*12 + 6] = R[0][1];
		posesData[i*12 + 7] = R[1][1];
		posesData[i*12 + 8] = R[2][1];
		posesData[i*12 + 9] = R[0][2];
		posesData[i*12 + 10] = R[1][2];
		posesData[i*12 + 11] = R[2][2];

		++i;
	}

	posesDataSet.write((const void*)posesData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL);
	free((void*)posesData);
	posesDataSet.close();
	posesDS.close();

	// Saving points
	const hsize_t pointsChunkDim[] = {10, 3};
	H5::DSetCreatPropList pointsPropList;
	pointsPropList.setLayout(H5D_CHUNKED);
	pointsPropList.setChunk(2, pointsChunkDim);
	pointsPropList.setDeflate(9);

	const hsize_t pointsMaxDim[] = { H5S_UNLIMITED, 3 };
	const hsize_t pointsCurDim[] = { tracks_.size(), 3 };
	H5::DataSpace pointsDS(2, pointsCurDim, pointsMaxDim);

	H5::DataSet pointsDataSet = geometryGroup.createDataSet("Points", H5::PredType::IEEE_F64LE, pointsDS, pointsPropList);

	double* pointsData = (double*)malloc(tracks_.size()*3*sizeof(double));

	i = 0;
	for(deque<Track*>::const_iterator it = tracks_.begin(); it != tracks_.end(); it++) {
		pointsData[i*3] = (*it)->point()->coords().x();
		pointsData[i*3 + 1] = (*it)->point()->coords().y();
		pointsData[i*3 + 2] = (*it)->point()->coords().z();

		++i;
	}

	pointsDataSet.write((const void*)pointsData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL);
	free((void*)pointsData);
	pointsDataSet.close();
	pointsDS.close();

	// Saving inlier information
	const hsize_t inliersChunkDim[] = { 3 };
	H5::DSetCreatPropList inliersPropList;
	inliersPropList.setLayout(H5D_CHUNKED);
	inliersPropList.setChunk(1, inliersChunkDim);
	inliersPropList.setDeflate(9);

	const hsize_t inliersMaxDim[] = { H5S_UNLIMITED };
	const hsize_t inliersCurDim[] = { frames_.size() };
	H5::DataSpace inliersDS(1, inliersCurDim, inliersMaxDim);

	H5::VarLenType inliersType(&H5::PredType::STD_U8LE);

	H5::DataSet inliersDataSet = geometryGroup.createDataSet("Inliers", inliersType, inliersDS, inliersPropList);

	i = 0;
	for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
		hvl_t inliersLine;

        size_t inliersLineSize = 0;
        for(size_t j = 0; j < (*it)->size(); ++j) {
            View& v = (**it)[j];
            for(unsigned int cam = 0; cam < v.numCameras(); ++cam) {
                if(v.inCamera(cam)) ++inliersLineSize;
            }
        }

		inliersLine.len = inliersLineSize;
		inliersLine.p = malloc(inliersLineSize*sizeof(unsigned char));

        size_t k = 0;
		for(size_t j = 0; j < (*it)->size(); ++j) {
            View& v = (**it)[j];
            for(unsigned int cam = 0; cam < v.numCameras(); ++cam) {
                if(v.inCamera(cam)) {
                    ((unsigned char*)(inliersLine.p))[k] = v.ray(cam).inlier()?1:0;
                    ++k;
                }
            }
        }

		const hsize_t dsOffset[] = { i };
		const hsize_t dsCount[] = { 1 };
		H5::DataSpace inliersCurDS = inliersDataSet.getSpace();
		inliersCurDS.selectHyperslab(H5S_SELECT_SET, dsCount, dsOffset);

		const hsize_t memDim[] = { 1 };
		H5::DataSpace memDS(1, memDim, memDim);

		H5::VarLenType memType(&H5::PredType::NATIVE_UCHAR);

		inliersDataSet.write((const void*)&inliersLine, memType, memDS, inliersCurDS);

		memType.close();
		memDS.close();
		inliersCurDS.close();

		free(inliersLine.p);

		++i;
	}

	inliersDataSet.close();
	inliersType.close();
	inliersDS.close();

	// Saving curves
	if(!curves_.empty()) {
		const hsize_t chunkDim[] = { 5 };
		H5::DSetCreatPropList propList;
		propList.setLayout(H5D_CHUNKED);
		propList.setChunk(1, chunkDim);
		propList.setDeflate(9);

		H5::VarLenType curveDatasetType(&H5::PredType::STD_U64LE);

		hsize_t curvesDim[] = { curves_.size() };
		hsize_t curvesMaxDim[] = { H5S_UNLIMITED };
		H5::DataSpace curvesDataspace(1, curvesDim, curvesMaxDim);

		H5::DataSet curvesDataset = geometryGroup.createDataSet("Curves", curveDatasetType, curvesDataspace, propList);

		for(size_t i = 0; i < curves_.size(); ++i) {
			hvl_t curveLine;

			curveLine.len = curves_[i].size();
			curveLine.p = malloc(curves_[i].size()*sizeof(size_t));

			for(size_t j = 0; j < curves_[i].size(); ++j) ((size_t*)(curveLine.p))[j] = curves_[i].track(j);

			const hsize_t dsOffset[] = { i };
			const hsize_t dsCount[] = { 1 };
			H5::DataSpace curDS = curvesDataset.getSpace();
			curDS.selectHyperslab(H5S_SELECT_SET, dsCount, dsOffset);

			const hsize_t memDim[] = { 1 };
			H5::DataSpace memDS(1, memDim, memDim);

			H5::VarLenType memType(&H5::PredType::NATIVE_HSIZE);

			curvesDataset.write((const void*)&curveLine, memType, memDS, curDS);

			memType.close();
			memDS.close();
			curDS.close();

			free(curveLine.p);
 		}

		curvesDataset.close();
		curvesDataspace.close();
		curveDatasetType.close();
		propList.close();
	}

	geometryGroup.close();
	rootGroup.close();
	bundleFile.close();
}