H5::DataSet CompartmentReportHDF5::_createDataset( const uint32_t gid,
const size_t compCount )
{
LBASSERT( compCount > 0 );
LBASSERT( !_reportName.empty( ));
std::ostringstream neuronName;
neuronName << "a" << gid;
H5::Group neuronGroup = _file.createGroup( neuronName.str().c_str( ));
H5::Group reportGroup = neuronGroup.createGroup( _reportName );
const int dims = 2;
const size_t numSteps = (getEndTime() - getStartTime()) / getTimestep();
const hsize_t mappingDim[dims] = { 1, compCount };
const hsize_t dataDim[dims] = { numSteps, compCount };
LBASSERT( numSteps > 0 );
H5::DataSpace mappingDataspace( dims, mappingDim );
H5::DataSpace dataDataspace( dims, dataDim );
H5::DataSet mappingDataset = reportGroup.createDataSet(
mappingDatasetName, H5::PredType::NATIVE_FLOAT, mappingDataspace );
H5::DataSet dataDataset = reportGroup.createDataSet( dataDatasetName,
H5::PredType::NATIVE_FLOAT, dataDataspace );
_datas[gid] = dataDataset;
_createMappingAttributes( mappingDataset );
_createDataAttributes( dataDataset );
return mappingDataset;
}
void HDF5IO::saveMatrix(const std::string& GroupName, const std::string& Name,
const ComplexMatrixType& M)
{
try{
H5::CompType ComplexDataType = this->openCompType("complex");
hsize_t Dims[2] = {hsize_t(M.rows()),hsize_t(M.cols())};
H5::DataSpace dataspace(2,Dims);
H5::Group FG = getGroup( GroupName );
try{
H5::Exception::dontPrint();
H5::DataSet dset = FG.openDataSet(Name.c_str());
// dset.extend( Dims );not working
dset.write(M.data(), ComplexDataType);
} catch ( const H5::GroupIException not_found_error ){
H5::DataSet dset = FG.createDataSet(Name.c_str(), ComplexDataType, dataspace);
dset.write(M.data(), ComplexDataType);
} catch ( const H5::DataSetIException error ){
error.printError();
RUNTIME_ERROR("HDF5IO::saveComplexMatrix at ");
}
FG.close();
} catch( const H5::Exception error ){
error.printError();
RUNTIME_ERROR("HDF5IO::saveComplexMatrix at ");
}
}
void writeArray(H5::Group &group, const std::string &name,
const std::string &value) {
StrType dataType(0, value.length() + 1);
DataSpace dataSpace = getDataSpace(1);
H5::DataSet data = group.createDataSet(name, dataType, dataSpace);
data.write(value, dataType);
}
void HDF5IO::saveStdVector(const std::string& GroupName, const std::string& Name,
const std::vector<std::complex<double> >& V)
{
try{
H5::CompType ComplexDataType = openCompType("complex");
hsize_t Dim[1] = {hsize_t(V.size())};
H5::DataSpace dataspace(1,Dim);
H5::Group FG = getGroup( GroupName.c_str() );
try{
H5::Exception::dontPrint();
H5::DataSet dataset = FG.openDataSet(Name.c_str());
dataset.write(V.data(), ComplexDataType, dataspace);
} catch( const H5::GroupIException not_found_error ){
H5::DataSet dataset = FG.createDataSet(Name.c_str(), ComplexDataType,
dataspace);
dataset.write(V.data(), ComplexDataType);
} catch( const H5::FileIException error){
error.printError();
} catch( const H5::DataSetIException error){
error.printError();
}
FG.close();
} catch( const H5::Exception err ){
err.printError();
RUNTIME_ERROR("HDF5IO::saveComplexStdVector. ");
}
}
H5::DataSet create_dataset(std::string const & key, Args && ... args) const {
unlink_key_if_exists(key);
H5::DataSet res;
try{ res = _g.createDataSet(key.c_str(), std::forward<Args>(args)...);}
catch (H5::GroupIException const & e){ TRIQS_RUNTIME_ERROR << "Error in creating the dataset "<< key <<"\n H5 error message : \n "<< e.getCDetailMsg(); }
return res;
}
void writeArray(H5::Group &group, const std::string &name,
const std::vector<int32_t> &values) {
DataType dataType(PredType::NATIVE_INT32);
DataSpace dataSpace = getDataSpace(values);
DSetCreatPropList propList = getPropList(values.size());
auto data = group.createDataSet(name, dataType, dataSpace, propList);
data.write(&(values[0]), dataType);
}
void HDF5IO::saveNumber(const std::string& GroupName, const std::string& Name,
unsigned long x)
{
H5::Group FG = getGroup( GroupName );
try{
H5::Exception::dontPrint();
H5::DataSet dataset = FG.openDataSet( Name.c_str() );
dataset.write(&x, H5::PredType::NATIVE_ULONG);
} catch ( const H5::GroupIException not_found_error ){
H5::DataSet dataset = FG.createDataSet( Name.c_str(), H5::PredType::NATIVE_ULONG, H5::DataSpace());
dataset.write(&x, H5::PredType::NATIVE_ULONG);
}
FG.close();
}
/* only for Eigen3 matrix/vector */
void HDF5IO::saveVector(const std::string& GroupName,
const std::string& Name, const RealVectorType& V)
{
hsize_t Dim[1] = {hsize_t(V.size())};
H5::DataSpace dspace(1,Dim);
H5::Group FG = getGroup( GroupName );
try{
H5::Exception::dontPrint();
H5::DataSet DataSet = FG.openDataSet(Name.c_str());
DataSet.write(V.data(),H5::PredType::NATIVE_DOUBLE, dspace);
} catch ( const H5::GroupIException not_found_error ){
H5::DataSet DataSet = FG.createDataSet(Name.c_str(),
H5::PredType::NATIVE_DOUBLE,dspace);
DataSet.write(V.data(),H5::PredType::NATIVE_DOUBLE);
}
FG.close();
}
void HDF5IO::saveNumber(const std::string& GroupName, const std::string& Name,
ComplexType C)
{
H5::CompType ComplexDataType = openCompType("complex");
H5::Group FG = getGroup( GroupName );
try{
H5::Exception::dontPrint();
H5::DataSet dataset = FG.openDataSet(Name.c_str());
RealType RealImag[2] = {real(C),imag(C)};
dataset.write(RealImag, ComplexDataType);
} catch ( const H5::GroupIException not_found_error ){
H5::DataSet dataset = FG.createDataSet(Name.c_str(), ComplexDataType, H5::DataSpace());
RealType RealImag[2] = {real(C),imag(C)};
dataset.write(RealImag, ComplexDataType);
}
FG.close();
}
void HDF5IO::saveMatrix(const std::string& GroupName, const std::string& Name,
const RealMatrixType& M)
{
try{
hsize_t Dims[2] = {hsize_t(M.rows()),hsize_t(M.cols())};
H5::DataSpace dataspace(2,Dims);
H5::Group FG = getGroup( GroupName );
try{
H5::Exception::dontPrint();
H5::DataSet DataSet = FG.openDataSet(Name.c_str());
DataSet.write(M.data(),H5::PredType::NATIVE_DOUBLE,dataspace);
} catch ( const H5::GroupIException not_found_error ){
H5::DataSet DataSet = FG.createDataSet(Name.c_str(),H5::PredType::NATIVE_DOUBLE,dataspace);
DataSet.write(M.data(),H5::PredType::NATIVE_DOUBLE);
}
FG.close();
} catch( const H5::Exception err ){
RUNTIME_ERROR("HDF5IO::saveRealMatrix");
}
}
void HDF5IO::saveStdVector(const std::string& GroupName, const std::string& Name,
const std::vector<double>& V)
{
try{
hsize_t Dim[1] = {hsize_t(V.size())};
H5::DataSpace dataspace(1,Dim);
H5::Group FG = getGroup( GroupName );
try{
H5::Exception::dontPrint();
H5::DataSet dataset = FG.openDataSet(Name.c_str());
dataset.write(V.data(),H5::PredType::NATIVE_DOUBLE, dataspace);
} catch ( const H5::GroupIException not_found_error ){
H5::DataSet dataset = FG.createDataSet(Name.c_str(),
H5::PredType::NATIVE_DOUBLE,dataspace);
dataset.write(V.data(),H5::PredType::NATIVE_DOUBLE);
}
FG.close();
} catch( const H5::Exception err ){
RUNTIME_ERROR("HDF5IO::saveRealStdVector");
}
}
void HDF5IO::saveVector(const std::string& GroupName,
const std::string& Name, const ComplexVectorType& V)
{
try{
H5::CompType ComplexDataType = this->openCompType("complex");
hsize_t Dim[1] = {hsize_t(V.size())};
H5::DataSpace dspace(1,Dim);
H5::Group FG = getGroup( GroupName );
try{
H5::Exception::dontPrint();
H5::DataSet DataSet = FG.openDataSet(Name.c_str());
DataSet.write(V.data(), ComplexDataType, dspace);
} catch ( const H5::GroupIException not_found_error ){
H5::DataSet DataSet = FG.createDataSet(Name.c_str(), ComplexDataType,
dspace);
DataSet.write(V.data(), ComplexDataType);
}
FG.close();
} catch ( const H5::DataSetIException error ){
error.printError();
RUNTIME_ERROR("HDF5IO::saveComplexVector at ");
}
}
void Bundle2::initFrameStream_() {
H5::DataSpace scalar;
// Creating datasets for each frame
H5::Group bundleGroup = streamFile_->createGroup("/Bundle");
H5::Group framesGroup = bundleGroup.createGroup("Frames");
hsize_t count = frames_.size();
H5::Attribute attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
// Defining frame dataset property
hsize_t chunkDim[] = { 1, 2, 10 };
H5::DSetCreatPropList propList;
propList.setLayout(H5D_CHUNKED);
propList.setChunk(3, chunkDim);
propList.setDeflate(9);
// Definig dataset dataspace
hsize_t max_dim[] = { numCameras_, 2, H5S_UNLIMITED };
hsize_t dim[] = { numCameras_, 2, 0 };
H5::DataSpace ds(3, dim, max_dim);
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
const std::string datasetName = boost::str(boost::format("Frame %1$04d") % (*it)->number());
// Creating dataset
H5::DataSet frameData = framesGroup.createDataSet(datasetName, H5::PredType::IEEE_F32LE, ds, propList);
// Writing global number
attr = frameData.createAttribute("globalNumber", H5::PredType::STD_U64LE, scalar);
count = (*it)->globalNumber();
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
// Clean up!
frameData.close();
}
// Clean up!
ds.close();
propList.close();
// Creating tracks dataset
hsize_t chunkDim2[] = { 2, 10 };
propList = H5::DSetCreatPropList();
propList.setLayout(H5D_CHUNKED);
propList.setChunk(2, chunkDim);
propList.setDeflate(9);
H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE);
hsize_t tracksDim[] = { 0, 2 };
hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 };
H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim);
H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList);
tracksDataset.close();
tracksDataspace.close();
tracksDatasetType.close();
propList.close();
framesGroup.close();
bundleGroup.close();
scalar.close();
}
bool saveStackHDF5( const char* fileName, const My4DImage& img, Codec_Mapping* mapping )
{
try
{
#ifdef USE_HDF5
H5::Exception::dontPrint();
H5::H5File file( fileName, H5F_ACC_TRUNC );
H5::Group* group = new H5::Group( file.createGroup( "/Channels" ) );
Image4DProxy<My4DImage> proxy( const_cast<My4DImage*>( &img ) );
long scaledHeight = nearestPowerOfEight( proxy.sy );
long scaledWidth = nearestPowerOfEight( proxy.sx );
// Initialize the upper and lower bounds
long pad_right = ( scaledWidth - proxy.sx ) ;
long pad_bottom = ( scaledHeight - proxy.sy );
hsize_t dims[1] = { 1 };
H5::DataSpace attr_ds = H5::DataSpace( 1, dims );
H5::Attribute attr = group->createAttribute( "width", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( proxy.sx ) );
attr = group->createAttribute( "height", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( proxy.sy ) );
attr = group->createAttribute( "frames", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( proxy.sz ) );
attr = group->createAttribute( "pad_right", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( pad_right ) );
attr = group->createAttribute( "pad_bottom", H5::PredType::STD_I64LE, attr_ds );
attr.write( H5::PredType::NATIVE_INT, &( pad_bottom ) );
Codec_Mapping* imap = mapping;
if ( !mapping )
{
imap = new Codec_Mapping();
generate_codec_mapping( *imap, proxy.sc );
}
for ( int c = 0; c < proxy.sc; ++c )
{
double default_irange = 1.0; // assumes data range is 0-255.0
if ( proxy.su > 1 )
{
default_irange = 1.0 / 16.0; // 0-4096, like our microscope images
}
std::vector<double> imin( proxy.sc, 0.0 );
std::vector<double> irange2( proxy.sc, default_irange );
// rescale if converting from 16 bit to 8 bit
if ( proxy.su > 1 )
{
if ( img.p_vmin && img.p_vmax )
proxy.set_minmax( img.p_vmin, img.p_vmax );
if ( proxy.has_minmax() )
{
imin[c] = proxy.vmin[c];
irange2[c] = 255.0 / ( proxy.vmax[c] - proxy.vmin[c] );
}
}
FFMpegEncoder encoder( NULL, scaledWidth, scaledHeight,
( *imap )[c].first, ( *imap )[c].second );
// If the image needs padding, fill the expanded border regions with black
for ( int z = 0; z < proxy.sz; ++z )
{
for ( int y = 0; y < scaledHeight; ++y )
{
for ( int x = 0; x < scaledWidth; ++x )
{
// If inside the area with valid data
if ( x < proxy.sx && y < proxy.sy )
{
int ic = c;
double val = proxy.value_at( x, y, z, ic );
val = ( val - imin[ic] ) * irange2[ic]; // rescale to range 0-255
for ( int cc = 0; cc < 3; ++cc )
encoder.setPixelIntensity( x, y, cc, ( int )val );
}
else
for ( int cc = 0; cc < 3; ++cc )
encoder.setPixelIntensity( x, y, cc, 0 );
}
}
encoder.write_frame();
}
for ( int rem = encoder.encoded_frames(); rem < proxy.sz; rem++ )
encoder.encode();
encoder.close();
hsize_t dims[1];
dims[0] = encoder.buffer_size();
H5::DataSpace dataspace( 1, dims );
std: stringstream name;
name << "Channel_" << c;
H5::DataSet dataset = group->createDataSet( name.str(), H5::PredType::NATIVE_UINT8, dataspace );
dataset.write( encoder.buffer(), H5::PredType::NATIVE_UINT8 );
dataset.close();
std::cout << "Encoded channel is " << encoder.buffer_size() << " bytes." << std::endl;
// Uncomment this if you want to dump the individual movies to a temp file
}
#endif
if ( !mapping )
delete imap;
file.close();
return true;
}
catch ( ... ) {}
return false;
}
bool TStellarData::save(const std::string& fname, const std::string& group, const std::string &dset, int compression) {
if((compression < 0) || (compression > 9)) {
std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
return false;
}
hsize_t nstars = star.size();
if(nstars == 0) {
std::cerr << "! No stars to write." << std::endl;
return false;
}
H5::Exception::dontPrint();
H5::H5File *file = H5Utils::openFile(fname);
if(file == NULL) { return false; }
H5::Group *gp = H5Utils::openGroup(file, group);
if(gp == NULL) {
delete file;
return false;
}
/*
* Photometry
*/
// Datatype
hsize_t nbands = NBANDS;
H5::ArrayType f4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
H5::ArrayType u4arr(H5::PredType::NATIVE_FLOAT, 1, &nbands);
H5::CompType dtype(sizeof(TFileData));
dtype.insertMember("obj_id", HOFFSET(TFileData, obj_id), H5::PredType::NATIVE_UINT64);
dtype.insertMember("l", HOFFSET(TFileData, l), H5::PredType::NATIVE_DOUBLE);
dtype.insertMember("b", HOFFSET(TFileData, b), H5::PredType::NATIVE_DOUBLE);
dtype.insertMember("mag", HOFFSET(TFileData, mag), f4arr);
dtype.insertMember("err", HOFFSET(TFileData, err), f4arr);
dtype.insertMember("maglimit", HOFFSET(TFileData, maglimit), f4arr);
dtype.insertMember("nDet", HOFFSET(TFileData, N_det), u4arr);
dtype.insertMember("EBV", HOFFSET(TFileData, EBV), H5::PredType::NATIVE_FLOAT);
// Dataspace
hsize_t dim = nstars;
H5::DataSpace dspace(1, &dim);
// Property List
H5::DSetCreatPropList plist;
plist.setChunk(1, &nstars);
plist.setDeflate(compression);
// Dataset
H5::DataSet dataset = gp->createDataSet(dset, dtype, dspace, plist);
// Write dataset
TFileData* data = new TFileData[nstars];
for(size_t i=0; i<nstars; i++) {
data[i].obj_id = star[i].obj_id;
data[i].l = star[i].l;
data[i].b = star[i].b;
for(size_t k=0; k<NBANDS; k++) {
data[i].mag[k] = star[i].m[k];
data[i].err[k] = star[i].err[k];
data[i].maglimit[k] = star[i].maglimit[k];
}
data[i].EBV = star[i].EBV;
}
dataset.write(data, dtype);
/*
* Attributes
*/
dim = 1;
H5::DataSpace att_dspace(1, &dim);
H5::PredType att_dtype = H5::PredType::NATIVE_UINT64;
H5::Attribute att_healpix_index = dataset.createAttribute("healpix_index", att_dtype, att_dspace);
att_healpix_index.write(att_dtype, &healpix_index);
att_dtype = H5::PredType::NATIVE_UINT32;
H5::Attribute att_nside = dataset.createAttribute("nside", att_dtype, att_dspace);
att_nside.write(att_dtype, &nside);
att_dtype = H5::PredType::NATIVE_UCHAR;
H5::Attribute att_nested = dataset.createAttribute("nested", att_dtype, att_dspace);
att_nested.write(att_dtype, &nested);
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_l = dataset.createAttribute("l", att_dtype, att_dspace);
att_l.write(att_dtype, &l);
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_b = dataset.createAttribute("b", att_dtype, att_dspace);
att_b.write(att_dtype, &b);
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_EBV = dataset.createAttribute("EBV", att_dtype, att_dspace);
att_EBV.write(att_dtype, &EBV);
file->close();
delete[] data;
delete gp;
delete file;
return true;
}
void Bundle2::saveGeometry(const boost::filesystem::path& fileName) const {
H5::H5File bundleFile;
bundleFile.openFile(fileName.string(), H5F_ACC_RDWR);
H5::Group rootGroup = bundleFile.openGroup("/");
// If the group "Geometry" exists, delete it!
if(checkGeometry_(bundleFile)) {
rootGroup.unlink("Geometry");
}
// Creating group Geometry
H5::Group geometryGroup = rootGroup.createGroup("Geometry");
// Saving poses
const hsize_t posesChunkDim[] = { 3, 12 };
H5::DSetCreatPropList posesPropList;
posesPropList.setLayout(H5D_CHUNKED);
posesPropList.setChunk(2, posesChunkDim);
posesPropList.setDeflate(9);
const hsize_t posesMaxDim[] = { H5S_UNLIMITED, 12 };
const hsize_t posesCurDim[] = { frames_.size(), 12 };
H5::DataSpace posesDS(2, posesCurDim, posesMaxDim);
H5::DataSet posesDataSet = geometryGroup.createDataSet("Poses", H5::PredType::IEEE_F64LE, posesDS, posesPropList);
double* posesData = (double*)malloc(frames_.size()*12*sizeof(double));
size_t i = 0;
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
posesData[i*12] = (*it)->pose()->t().x();
posesData[i*12 + 1] = (*it)->pose()->t().y();
posesData[i*12 + 2] = (*it)->pose()->t().z();
core::Matrix<double> R = (*it)->pose()->R();
posesData[i*12 + 3] = R[0][0];
posesData[i*12 + 4] = R[1][0];
posesData[i*12 + 5] = R[2][0];
posesData[i*12 + 6] = R[0][1];
posesData[i*12 + 7] = R[1][1];
posesData[i*12 + 8] = R[2][1];
posesData[i*12 + 9] = R[0][2];
posesData[i*12 + 10] = R[1][2];
posesData[i*12 + 11] = R[2][2];
++i;
}
posesDataSet.write((const void*)posesData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL);
free((void*)posesData);
posesDataSet.close();
posesDS.close();
// Saving points
const hsize_t pointsChunkDim[] = {10, 3};
H5::DSetCreatPropList pointsPropList;
pointsPropList.setLayout(H5D_CHUNKED);
pointsPropList.setChunk(2, pointsChunkDim);
pointsPropList.setDeflate(9);
const hsize_t pointsMaxDim[] = { H5S_UNLIMITED, 3 };
const hsize_t pointsCurDim[] = { tracks_.size(), 3 };
H5::DataSpace pointsDS(2, pointsCurDim, pointsMaxDim);
H5::DataSet pointsDataSet = geometryGroup.createDataSet("Points", H5::PredType::IEEE_F64LE, pointsDS, pointsPropList);
double* pointsData = (double*)malloc(tracks_.size()*3*sizeof(double));
i = 0;
for(deque<Track*>::const_iterator it = tracks_.begin(); it != tracks_.end(); it++) {
pointsData[i*3] = (*it)->point()->coords().x();
pointsData[i*3 + 1] = (*it)->point()->coords().y();
pointsData[i*3 + 2] = (*it)->point()->coords().z();
++i;
}
pointsDataSet.write((const void*)pointsData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL);
free((void*)pointsData);
pointsDataSet.close();
pointsDS.close();
// Saving inlier information
const hsize_t inliersChunkDim[] = { 3 };
H5::DSetCreatPropList inliersPropList;
inliersPropList.setLayout(H5D_CHUNKED);
inliersPropList.setChunk(1, inliersChunkDim);
inliersPropList.setDeflate(9);
const hsize_t inliersMaxDim[] = { H5S_UNLIMITED };
const hsize_t inliersCurDim[] = { frames_.size() };
H5::DataSpace inliersDS(1, inliersCurDim, inliersMaxDim);
H5::VarLenType inliersType(&H5::PredType::STD_U8LE);
H5::DataSet inliersDataSet = geometryGroup.createDataSet("Inliers", inliersType, inliersDS, inliersPropList);
i = 0;
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
hvl_t inliersLine;
size_t inliersLineSize = 0;
for(size_t j = 0; j < (*it)->size(); ++j) {
View& v = (**it)[j];
for(unsigned int cam = 0; cam < v.numCameras(); ++cam) {
if(v.inCamera(cam)) ++inliersLineSize;
}
}
inliersLine.len = inliersLineSize;
inliersLine.p = malloc(inliersLineSize*sizeof(unsigned char));
size_t k = 0;
for(size_t j = 0; j < (*it)->size(); ++j) {
View& v = (**it)[j];
for(unsigned int cam = 0; cam < v.numCameras(); ++cam) {
if(v.inCamera(cam)) {
((unsigned char*)(inliersLine.p))[k] = v.ray(cam).inlier()?1:0;
++k;
}
}
}
const hsize_t dsOffset[] = { i };
const hsize_t dsCount[] = { 1 };
H5::DataSpace inliersCurDS = inliersDataSet.getSpace();
inliersCurDS.selectHyperslab(H5S_SELECT_SET, dsCount, dsOffset);
const hsize_t memDim[] = { 1 };
H5::DataSpace memDS(1, memDim, memDim);
H5::VarLenType memType(&H5::PredType::NATIVE_UCHAR);
inliersDataSet.write((const void*)&inliersLine, memType, memDS, inliersCurDS);
memType.close();
memDS.close();
inliersCurDS.close();
free(inliersLine.p);
++i;
}
inliersDataSet.close();
inliersType.close();
inliersDS.close();
// Saving curves
if(!curves_.empty()) {
const hsize_t chunkDim[] = { 5 };
H5::DSetCreatPropList propList;
propList.setLayout(H5D_CHUNKED);
propList.setChunk(1, chunkDim);
propList.setDeflate(9);
H5::VarLenType curveDatasetType(&H5::PredType::STD_U64LE);
hsize_t curvesDim[] = { curves_.size() };
hsize_t curvesMaxDim[] = { H5S_UNLIMITED };
H5::DataSpace curvesDataspace(1, curvesDim, curvesMaxDim);
H5::DataSet curvesDataset = geometryGroup.createDataSet("Curves", curveDatasetType, curvesDataspace, propList);
for(size_t i = 0; i < curves_.size(); ++i) {
hvl_t curveLine;
curveLine.len = curves_[i].size();
curveLine.p = malloc(curves_[i].size()*sizeof(size_t));
for(size_t j = 0; j < curves_[i].size(); ++j) ((size_t*)(curveLine.p))[j] = curves_[i].track(j);
const hsize_t dsOffset[] = { i };
const hsize_t dsCount[] = { 1 };
H5::DataSpace curDS = curvesDataset.getSpace();
curDS.selectHyperslab(H5S_SELECT_SET, dsCount, dsOffset);
const hsize_t memDim[] = { 1 };
H5::DataSpace memDS(1, memDim, memDim);
H5::VarLenType memType(&H5::PredType::NATIVE_HSIZE);
curvesDataset.write((const void*)&curveLine, memType, memDS, curDS);
memType.close();
memDS.close();
curDS.close();
free(curveLine.p);
}
curvesDataset.close();
curvesDataspace.close();
curveDatasetType.close();
propList.close();
}
geometryGroup.close();
rootGroup.close();
bundleFile.close();
}
// Bundle management
void Bundle2::save(const boost::filesystem::path& fileName) const {
// Creating HDF5 file
H5::H5File bundleFile(fileName.string(), H5F_ACC_TRUNC);
storeParameters(bundleFile);
H5::DataSpace scalar;
// Saving POI
H5::Group poiGroup = bundleFile.createGroup("/POI");
H5::Attribute attr = poiGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
hsize_t count = poi_.size();
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
for(size_t frame = 0; frame < poi_.size(); ++frame) {
const std::string frameGroupName = boost::str(boost::format("Frame %1$04d") % frame);
H5::Group frameGroup = poiGroup.createGroup(frameGroupName);
count = poi_[frame].size();
attr = frameGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
for(size_t camera = 0; camera < poi_[frame].size(); ++camera)
poi_[frame][camera].save(frameGroup, camera);
frameGroup.close();
}
poiGroup.close();
// Saving key frames
H5::Group bundleGroup = bundleFile.createGroup("/Bundle");
H5::Group framesGroup = bundleGroup.createGroup("Frames");
count = frames_.size();
attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar);
attr.write(H5::PredType::NATIVE_HSIZE, &count);
attr.close();
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
(*it)->save(framesGroup);
}
framesGroup.close();
// Saving tracks
const hsize_t chunkDim[] = { 2, 1 };
H5::DSetCreatPropList propList;
propList.setLayout(H5D_CHUNKED);
propList.setChunk(2, chunkDim);
propList.setDeflate(9);
H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE);
hsize_t tracksDim[] = { tracks_.size(), 2 };
hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 };
H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim);
H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList);
for(size_t i = 0; i < tracks_.size(); ++i)
tracks_[i]->save(tracksDataset, i);
tracksDataset.close();
tracksDataspace.close();
tracksDatasetType.close();
propList.close();
bundleGroup.close();
scalar.close();
bundleFile.close();
}
bool save_mat_image(cv::Mat& img, TRect& rect, std::string fname, std::string group_name,
std::string dset_name, std::string dim1, std::string dim2, int compression) {
assert((img.dims == 2) && (img.rows == rect.N_bins[0]) && (img.cols == rect.N_bins[1]));
if((compression<0) || (compression > 9)) {
std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
return false;
}
H5::Exception::dontPrint();
H5::H5File *file = H5Utils::openFile(fname);
if(file == NULL) { return false; }
H5::Group *group = H5Utils::openGroup(file, group_name);
if(group == NULL) {
delete file;
return false;
}
/*
* Image Data
*/
// Creation property list
H5::DSetCreatPropList plist;
int rank = 2;
hsize_t dim[2] = {rect.N_bins[0], rect.N_bins[1]};
plist.setDeflate(compression); // gzip compression level
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
plist.setChunk(rank, &(dim[0]));
H5::DataSpace dspace(rank, &(dim[0]));
H5::DataSet* dataset;
try {
dataset = new H5::DataSet(group->createDataSet(dset_name, H5::PredType::NATIVE_FLOAT, dspace, plist));
} catch(H5::FileIException create_dset_err) {
std::cerr << "Unable to create dataset '" << dset_name << "'." << std::endl;
delete group;
delete file;
return false;
}
float *buf = new float[rect.N_bins[0]*rect.N_bins[1]];
for(size_t j=0; j<rect.N_bins[0]; j++) {
for(size_t k=0; k<rect.N_bins[1]; k++) {
buf[rect.N_bins[1]*j + k] = img.at<double>(j,k);
/*float tmp = img.at<double>(j,k);
if(tmp > 0.) {
std::cerr << j << ", " << k << " --> " << j + rect.N_bins[0]*k << " --> " << tmp << std::endl;
}*/
}
}
dataset->write(buf, H5::PredType::NATIVE_FLOAT);
/*
* Attributes
*/
hsize_t att_dim = 2;
H5::DataSpace att_dspace(1, &att_dim);
H5::PredType att_dtype = H5::PredType::NATIVE_UINT32;
H5::Attribute att_N = dataset->createAttribute("N_pix", att_dtype, att_dspace);
att_N.write(att_dtype, &(rect.N_bins));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_min = dataset->createAttribute("min", att_dtype, att_dspace);
att_min.write(att_dtype, &(rect.min));
att_dtype = H5::PredType::NATIVE_DOUBLE;
H5::Attribute att_max = dataset->createAttribute("max", att_dtype, att_dspace);
att_max.write(att_dtype, &(rect.max));
att_dim = 1;
H5::StrType vls_type(0, H5T_VARIABLE);
H5::DataSpace att_space_str(H5S_SCALAR);
H5::Attribute att_name_1 = dataset->createAttribute("dim_name_1", vls_type, att_space_str);
att_name_1.write(vls_type, dim1);
H5::Attribute att_name_2 = dataset->createAttribute("dim_name_2", vls_type, att_space_str);
att_name_2.write(vls_type, dim2);
file->close();
delete[] buf;
delete dataset;
delete group;
delete file;
return true;
}
bool TChain::save(std::string fname, std::string group_name, size_t index,
std::string dim_name, int compression, int subsample,
bool converged, float lnZ) const {
if((compression<0) || (compression > 9)) {
std::cerr << "! Invalid gzip compression level: " << compression << std::endl;
return false;
}
H5::Exception::dontPrint();
H5::H5File *file = H5Utils::openFile(fname);
if(file == NULL) { return false; }
/*
try {
file->unlink(group_name);
} catch(...) {
// pass
}
*/
H5::Group *group = H5Utils::openGroup(file, group_name);
if(group == NULL) {
delete file;
return false;
}
/*
* Attributes
*/
// Datatype
H5::CompType att_type(sizeof(TChainAttribute));
hid_t tid = H5Tcopy(H5T_C_S1);
H5Tset_size(tid, H5T_VARIABLE);
att_type.insertMember("dim_name", HOFFSET(TChainAttribute, dim_name), tid);
//att_type.insertMember("total_weight", HOFFSET(TChainAttribute, total_weight), H5::PredType::NATIVE_FLOAT);
//att_type.insertMember("ndim", HOFFSET(TChainAttribute, ndim), H5::PredType::NATIVE_UINT64);
//att_type.insertMember("length", HOFFSET(TChainAttribute, length), H5::PredType::NATIVE_UINT64);
// Dataspace
int att_rank = 1;
hsize_t att_dim = 1;
H5::DataSpace att_space(att_rank, &att_dim);
// Dataset
//H5::Attribute att = group->createAttribute("parameter names", att_type, att_space);
TChainAttribute att_data;
att_data.dim_name = new char[dim_name.size()+1];
std::strcpy(att_data.dim_name, dim_name.c_str());
//att_data.total_weight = total_weight;
//att_data.ndim = N;
//att_data.length = length;
//att.write(att_type, &att_data);
delete[] att_data.dim_name;
//int att_rank = 1;
//hsize_t att_dim = 1;
H5::DataType conv_dtype = H5::PredType::NATIVE_UCHAR;
H5::DataSpace conv_dspace(att_rank, &att_dim);
//H5::Attribute conv_att = H5Utils::openAttribute(group, "converged", conv_dtype, conv_dspace);
//conv_att.write(conv_dtype, &converged);
H5::DataType lnZ_dtype = H5::PredType::NATIVE_FLOAT;
H5::DataSpace lnZ_dspace(att_rank, &att_dim);
//H5::Attribute lnZ_att = H5Utils::openAttribute(group, "ln Z", lnZ_dtype, lnZ_dspace);
//lnZ_att.write(lnZ_dtype, &lnZ);
// Creation property list to be used for all three datasets
H5::DSetCreatPropList plist;
//plist.setDeflate(compression); // gzip compression level
float fillvalue = 0;
plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue);
H5D_layout_t layout = H5D_COMPACT;
plist.setLayout(layout);
/*
* Choose subsample of points in chain
*/
size_t *el_idx = NULL;
size_t *subsample_idx = NULL;
if(subsample > 0) {
size_t tot_weight_tmp = (size_t)ceil(total_weight);
el_idx = new size_t[tot_weight_tmp];
size_t unrolled_idx = 0;
size_t chain_idx = 0;
std::vector<double>::const_iterator it, it_end;
it_end = w.end();
for(it = w.begin(); it != it_end; ++it, chain_idx++) {
for(size_t n = unrolled_idx; n < unrolled_idx + (size_t)(*it); n++) {
el_idx[n] = chain_idx;
}
unrolled_idx += (size_t)(*it);
}
assert(chain_idx == length);
gsl_rng *r;
seed_gsl_rng(&r);
subsample_idx = new size_t[tot_weight_tmp];
for(size_t i=0; i<subsample; i++) {
subsample_idx[i] = el_idx[gsl_rng_uniform_int(r, tot_weight_tmp)];
}
}
/*
* Coordinates
*/
// Dataspace
hsize_t dim;
if(subsample > 0) {
dim = subsample;
} else {
dim = length;
}
// Chunking (required for compression)
int rank = 2;
hsize_t coord_dim[2] = {dim, N};
//if(dim < chunk) {
//plist.setChunk(rank, &(coord_dim[0]));
//} else {
// plist.setChunk(rank, &chunk);
//}
H5::DataSpace x_dspace(rank, &(coord_dim[0]));
// Dataset
//std::stringstream x_dset_path;
//x_dset_path << group_name << "/chain/coords";
std::stringstream coordname;
coordname << "coords " << index;
H5::DataSet* x_dataset = new H5::DataSet(group->createDataSet(coordname.str(), H5::PredType::NATIVE_FLOAT, x_dspace, plist));
// Write
float *buf = new float[N*dim];
if(subsample > 0) {
size_t tmp_idx;
for(size_t i=0; i<subsample; i++) {
tmp_idx = subsample_idx[i];
for(size_t k=0; k<N; k++) {
buf[N*i+k] = x[N*tmp_idx+k];
}
}
} else {
for(size_t i=0; i<dim; i++) { buf[i] = x[i]; }
}
x_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
/*
* Weights
*/
// Dataspace
if(subsample <= 0) {
dim = w.size();
rank = 1;
H5::DataSpace w_dspace(rank, &dim);
// Dataset
//std::stringstream w_dset_path;
//w_dset_path << group_name << "/chain/weights";
H5::DataSet* w_dataset = new H5::DataSet(group->createDataSet("weights", H5::PredType::NATIVE_FLOAT, w_dspace, plist));
// Write
if(subsample > 0) {
for(size_t i=0; i<subsample; i++) { buf[i] = 1.; }
} else {
assert(w.size() < x.size());
for(size_t i=0; i<w.size(); i++) { buf[i] = w[i]; }
}
w_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
delete w_dataset;
}
/*
* Probability densities
*/
// Dataspace
rank = 1;
H5::DataSpace L_dspace(rank, &dim);
// Dataset
//std::stringstream L_dset_path;
//L_dset_path << group_name << "/chain/probs";
std::stringstream lnpname;
lnpname << "ln_p " << index;
H5::DataSet* L_dataset = new H5::DataSet(group->createDataSet(lnpname.str(), H5::PredType::NATIVE_FLOAT, L_dspace, plist));
// Write
if(subsample > 0) {
for(size_t i=0; i<subsample; i++) { buf[i] = L[subsample_idx[i]]; }
} else {
assert(L.size() < x.size());
for(size_t i=0; i<L.size(); i++) { buf[i] = L[i]; }
}
L_dataset->write(buf, H5::PredType::NATIVE_FLOAT);
if(subsample > 0) {
delete[] el_idx;
delete[] subsample_idx;
}
delete[] buf;
delete x_dataset;
delete L_dataset;
delete group;
delete file;
return true;
}
void Bundle2::initGeometryStream_() {
// Creating group Geometry
H5::Group geometryGroup = streamFile_->createGroup("/Geometry");
// Saving poses
const hsize_t posesChunkDim[] = { 3, 12 };
H5::DSetCreatPropList posesPropList;
posesPropList.setLayout(H5D_CHUNKED);
posesPropList.setChunk(2, posesChunkDim);
posesPropList.setDeflate(9);
const hsize_t posesMaxDim[] = { H5S_UNLIMITED, 12 };
const hsize_t posesCurDim[] = { frames_.size(), 12 };
H5::DataSpace posesDS(2, posesCurDim, posesMaxDim);
H5::DataSet posesDataSet = geometryGroup.createDataSet("Poses", H5::PredType::IEEE_F64LE, posesDS, posesPropList);
double* posesData = (double*)malloc(frames_.size()*12*sizeof(double));
size_t i = 0;
for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) {
posesData[i*12] = (*it)->pose()->t().x();
posesData[i*12 + 1] = (*it)->pose()->t().y();
posesData[i*12 + 2] = (*it)->pose()->t().z();
core::Matrix<double> R = (*it)->pose()->R();
posesData[i*12 + 3] = R[0][0];
posesData[i*12 + 4] = R[1][0];
posesData[i*12 + 5] = R[2][0];
posesData[i*12 + 6] = R[0][1];
posesData[i*12 + 7] = R[1][1];
posesData[i*12 + 8] = R[2][1];
posesData[i*12 + 9] = R[0][2];
posesData[i*12 + 10] = R[1][2];
posesData[i*12 + 11] = R[2][2];
++i;
}
posesDataSet.write((const void*)posesData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL);
free((void*)posesData);
posesDataSet.close();
posesDS.close();
// Creating points dataset
const hsize_t pointsChunkDim[] = {10, 3};
H5::DSetCreatPropList pointsPropList;
pointsPropList.setLayout(H5D_CHUNKED);
pointsPropList.setChunk(2, pointsChunkDim);
pointsPropList.setDeflate(9);
const hsize_t pointsMaxDim[] = { H5S_UNLIMITED, 3 };
const hsize_t pointsCurDim[] = { 0, 3 };
H5::DataSpace pointsDS(2, pointsCurDim, pointsMaxDim);
H5::DataSet pointsDataSet = geometryGroup.createDataSet("Points", H5::PredType::IEEE_F64LE, pointsDS, pointsPropList);
pointsDataSet.close();
pointsDS.close();
// Creating inliers dataset
const hsize_t inliersChunkDim[] = { 3 };
H5::DSetCreatPropList inliersPropList;
inliersPropList.setLayout(H5D_CHUNKED);
inliersPropList.setChunk(1, inliersChunkDim);
inliersPropList.setDeflate(9);
const hsize_t inliersMaxDim[] = { H5S_UNLIMITED };
const hsize_t inliersCurDim[] = { frames_.size() };
H5::DataSpace inliersDS(1, inliersCurDim, inliersMaxDim);
H5::VarLenType inliersType(&H5::PredType::STD_U8LE);
H5::DataSet inliersDataSet = geometryGroup.createDataSet("Inliers", inliersType, inliersDS, inliersPropList);
inliersDataSet.close();
inliersType.close();
inliersDS.close();
// Creating curve dataset
const hsize_t chunkDim[] = { 5 };
H5::DSetCreatPropList propList;
propList.setLayout(H5D_CHUNKED);
propList.setChunk(1, chunkDim);
propList.setDeflate(9);
H5::VarLenType curveDatasetType(&H5::PredType::STD_U64LE);
hsize_t curvesDim[] = { 0 };
hsize_t curvesMaxDim[] = { H5S_UNLIMITED };
H5::DataSpace curvesDataspace(1, curvesDim, curvesMaxDim);
H5::DataSet curvesDataset = geometryGroup.createDataSet("Curves", curveDatasetType, curvesDataspace, propList);
curvesDataset.close();
curvesDataspace.close();
curveDatasetType.close();
propList.close();
geometryGroup.close();
}