bool TChain::save(std::string fname, std::string group_name, size_t index, std::string dim_name, int compression, int subsample, bool converged, float lnZ) const { if((compression<0) || (compression > 9)) { std::cerr << "! Invalid gzip compression level: " << compression << std::endl; return false; } H5::Exception::dontPrint(); H5::H5File *file = H5Utils::openFile(fname); if(file == NULL) { return false; } /* try { file->unlink(group_name); } catch(...) { // pass } */ H5::Group *group = H5Utils::openGroup(file, group_name); if(group == NULL) { delete file; return false; } /* * Attributes */ // Datatype H5::CompType att_type(sizeof(TChainAttribute)); hid_t tid = H5Tcopy(H5T_C_S1); H5Tset_size(tid, H5T_VARIABLE); att_type.insertMember("dim_name", HOFFSET(TChainAttribute, dim_name), tid); //att_type.insertMember("total_weight", HOFFSET(TChainAttribute, total_weight), H5::PredType::NATIVE_FLOAT); //att_type.insertMember("ndim", HOFFSET(TChainAttribute, ndim), H5::PredType::NATIVE_UINT64); //att_type.insertMember("length", HOFFSET(TChainAttribute, length), H5::PredType::NATIVE_UINT64); // Dataspace int att_rank = 1; hsize_t att_dim = 1; H5::DataSpace att_space(att_rank, &att_dim); // Dataset //H5::Attribute att = group->createAttribute("parameter names", att_type, att_space); TChainAttribute att_data; att_data.dim_name = new char[dim_name.size()+1]; std::strcpy(att_data.dim_name, dim_name.c_str()); //att_data.total_weight = total_weight; //att_data.ndim = N; //att_data.length = length; //att.write(att_type, &att_data); delete[] att_data.dim_name; //int att_rank = 1; //hsize_t att_dim = 1; H5::DataType conv_dtype = H5::PredType::NATIVE_UCHAR; H5::DataSpace conv_dspace(att_rank, &att_dim); //H5::Attribute conv_att = H5Utils::openAttribute(group, "converged", conv_dtype, conv_dspace); //conv_att.write(conv_dtype, &converged); H5::DataType lnZ_dtype = H5::PredType::NATIVE_FLOAT; H5::DataSpace lnZ_dspace(att_rank, &att_dim); //H5::Attribute lnZ_att = H5Utils::openAttribute(group, "ln Z", lnZ_dtype, lnZ_dspace); //lnZ_att.write(lnZ_dtype, &lnZ); // Creation property list to be used for all three datasets H5::DSetCreatPropList plist; //plist.setDeflate(compression); // gzip compression level float fillvalue = 0; plist.setFillValue(H5::PredType::NATIVE_FLOAT, &fillvalue); H5D_layout_t layout = H5D_COMPACT; plist.setLayout(layout); /* * Choose subsample of points in chain */ size_t *el_idx = NULL; size_t *subsample_idx = NULL; if(subsample > 0) { size_t tot_weight_tmp = (size_t)ceil(total_weight); el_idx = new size_t[tot_weight_tmp]; size_t unrolled_idx = 0; size_t chain_idx = 0; std::vector<double>::const_iterator it, it_end; it_end = w.end(); for(it = w.begin(); it != it_end; ++it, chain_idx++) { for(size_t n = unrolled_idx; n < unrolled_idx + (size_t)(*it); n++) { el_idx[n] = chain_idx; } unrolled_idx += (size_t)(*it); } assert(chain_idx == length); gsl_rng *r; seed_gsl_rng(&r); subsample_idx = new size_t[tot_weight_tmp]; for(size_t i=0; i<subsample; i++) { subsample_idx[i] = el_idx[gsl_rng_uniform_int(r, tot_weight_tmp)]; } } /* * Coordinates */ // Dataspace hsize_t dim; if(subsample > 0) { dim = subsample; } else { dim = length; } // Chunking (required for compression) int rank = 2; hsize_t coord_dim[2] = {dim, N}; //if(dim < chunk) { //plist.setChunk(rank, &(coord_dim[0])); //} else { // plist.setChunk(rank, &chunk); //} H5::DataSpace x_dspace(rank, &(coord_dim[0])); // Dataset //std::stringstream x_dset_path; //x_dset_path << group_name << "/chain/coords"; std::stringstream coordname; coordname << "coords " << index; H5::DataSet* x_dataset = new H5::DataSet(group->createDataSet(coordname.str(), H5::PredType::NATIVE_FLOAT, x_dspace, plist)); // Write float *buf = new float[N*dim]; if(subsample > 0) { size_t tmp_idx; for(size_t i=0; i<subsample; i++) { tmp_idx = subsample_idx[i]; for(size_t k=0; k<N; k++) { buf[N*i+k] = x[N*tmp_idx+k]; } } } else { for(size_t i=0; i<dim; i++) { buf[i] = x[i]; } } x_dataset->write(buf, H5::PredType::NATIVE_FLOAT); /* * Weights */ // Dataspace if(subsample <= 0) { dim = w.size(); rank = 1; H5::DataSpace w_dspace(rank, &dim); // Dataset //std::stringstream w_dset_path; //w_dset_path << group_name << "/chain/weights"; H5::DataSet* w_dataset = new H5::DataSet(group->createDataSet("weights", H5::PredType::NATIVE_FLOAT, w_dspace, plist)); // Write if(subsample > 0) { for(size_t i=0; i<subsample; i++) { buf[i] = 1.; } } else { assert(w.size() < x.size()); for(size_t i=0; i<w.size(); i++) { buf[i] = w[i]; } } w_dataset->write(buf, H5::PredType::NATIVE_FLOAT); delete w_dataset; } /* * Probability densities */ // Dataspace rank = 1; H5::DataSpace L_dspace(rank, &dim); // Dataset //std::stringstream L_dset_path; //L_dset_path << group_name << "/chain/probs"; std::stringstream lnpname; lnpname << "ln_p " << index; H5::DataSet* L_dataset = new H5::DataSet(group->createDataSet(lnpname.str(), H5::PredType::NATIVE_FLOAT, L_dspace, plist)); // Write if(subsample > 0) { for(size_t i=0; i<subsample; i++) { buf[i] = L[subsample_idx[i]]; } } else { assert(L.size() < x.size()); for(size_t i=0; i<L.size(); i++) { buf[i] = L[i]; } } L_dataset->write(buf, H5::PredType::NATIVE_FLOAT); if(subsample > 0) { delete[] el_idx; delete[] subsample_idx; } delete[] buf; delete x_dataset; delete L_dataset; delete group; delete file; return true; }
// Bundle management void Bundle2::save(const boost::filesystem::path& fileName) const { // Creating HDF5 file H5::H5File bundleFile(fileName.string(), H5F_ACC_TRUNC); storeParameters(bundleFile); H5::DataSpace scalar; // Saving POI H5::Group poiGroup = bundleFile.createGroup("/POI"); H5::Attribute attr = poiGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar); hsize_t count = poi_.size(); attr.write(H5::PredType::NATIVE_HSIZE, &count); attr.close(); for(size_t frame = 0; frame < poi_.size(); ++frame) { const std::string frameGroupName = boost::str(boost::format("Frame %1$04d") % frame); H5::Group frameGroup = poiGroup.createGroup(frameGroupName); count = poi_[frame].size(); attr = frameGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar); attr.write(H5::PredType::NATIVE_HSIZE, &count); attr.close(); for(size_t camera = 0; camera < poi_[frame].size(); ++camera) poi_[frame][camera].save(frameGroup, camera); frameGroup.close(); } poiGroup.close(); // Saving key frames H5::Group bundleGroup = bundleFile.createGroup("/Bundle"); H5::Group framesGroup = bundleGroup.createGroup("Frames"); count = frames_.size(); attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar); attr.write(H5::PredType::NATIVE_HSIZE, &count); attr.close(); for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) { (*it)->save(framesGroup); } framesGroup.close(); // Saving tracks const hsize_t chunkDim[] = { 2, 1 }; H5::DSetCreatPropList propList; propList.setLayout(H5D_CHUNKED); propList.setChunk(2, chunkDim); propList.setDeflate(9); H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE); hsize_t tracksDim[] = { tracks_.size(), 2 }; hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 }; H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim); H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList); for(size_t i = 0; i < tracks_.size(); ++i) tracks_[i]->save(tracksDataset, i); tracksDataset.close(); tracksDataspace.close(); tracksDatasetType.close(); propList.close(); bundleGroup.close(); scalar.close(); bundleFile.close(); }
void Bundle2::initFrameStream_() { H5::DataSpace scalar; // Creating datasets for each frame H5::Group bundleGroup = streamFile_->createGroup("/Bundle"); H5::Group framesGroup = bundleGroup.createGroup("Frames"); hsize_t count = frames_.size(); H5::Attribute attr = framesGroup.createAttribute("count", H5::PredType::STD_U64LE, scalar); attr.write(H5::PredType::NATIVE_HSIZE, &count); attr.close(); // Defining frame dataset property hsize_t chunkDim[] = { 1, 2, 10 }; H5::DSetCreatPropList propList; propList.setLayout(H5D_CHUNKED); propList.setChunk(3, chunkDim); propList.setDeflate(9); // Definig dataset dataspace hsize_t max_dim[] = { numCameras_, 2, H5S_UNLIMITED }; hsize_t dim[] = { numCameras_, 2, 0 }; H5::DataSpace ds(3, dim, max_dim); for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) { const std::string datasetName = boost::str(boost::format("Frame %1$04d") % (*it)->number()); // Creating dataset H5::DataSet frameData = framesGroup.createDataSet(datasetName, H5::PredType::IEEE_F32LE, ds, propList); // Writing global number attr = frameData.createAttribute("globalNumber", H5::PredType::STD_U64LE, scalar); count = (*it)->globalNumber(); attr.write(H5::PredType::NATIVE_HSIZE, &count); attr.close(); // Clean up! frameData.close(); } // Clean up! ds.close(); propList.close(); // Creating tracks dataset hsize_t chunkDim2[] = { 2, 10 }; propList = H5::DSetCreatPropList(); propList.setLayout(H5D_CHUNKED); propList.setChunk(2, chunkDim); propList.setDeflate(9); H5::VarLenType tracksDatasetType(&H5::PredType::STD_U64LE); hsize_t tracksDim[] = { 0, 2 }; hsize_t tracksMaxDim[] = { H5S_UNLIMITED, 2 }; H5::DataSpace tracksDataspace(2, tracksDim, tracksMaxDim); H5::DataSet tracksDataset = bundleGroup.createDataSet("Tracks", tracksDatasetType, tracksDataspace, propList); tracksDataset.close(); tracksDataspace.close(); tracksDatasetType.close(); propList.close(); framesGroup.close(); bundleGroup.close(); scalar.close(); }
void Bundle2::initGeometryStream_() { // Creating group Geometry H5::Group geometryGroup = streamFile_->createGroup("/Geometry"); // Saving poses const hsize_t posesChunkDim[] = { 3, 12 }; H5::DSetCreatPropList posesPropList; posesPropList.setLayout(H5D_CHUNKED); posesPropList.setChunk(2, posesChunkDim); posesPropList.setDeflate(9); const hsize_t posesMaxDim[] = { H5S_UNLIMITED, 12 }; const hsize_t posesCurDim[] = { frames_.size(), 12 }; H5::DataSpace posesDS(2, posesCurDim, posesMaxDim); H5::DataSet posesDataSet = geometryGroup.createDataSet("Poses", H5::PredType::IEEE_F64LE, posesDS, posesPropList); double* posesData = (double*)malloc(frames_.size()*12*sizeof(double)); size_t i = 0; for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) { posesData[i*12] = (*it)->pose()->t().x(); posesData[i*12 + 1] = (*it)->pose()->t().y(); posesData[i*12 + 2] = (*it)->pose()->t().z(); core::Matrix<double> R = (*it)->pose()->R(); posesData[i*12 + 3] = R[0][0]; posesData[i*12 + 4] = R[1][0]; posesData[i*12 + 5] = R[2][0]; posesData[i*12 + 6] = R[0][1]; posesData[i*12 + 7] = R[1][1]; posesData[i*12 + 8] = R[2][1]; posesData[i*12 + 9] = R[0][2]; posesData[i*12 + 10] = R[1][2]; posesData[i*12 + 11] = R[2][2]; ++i; } posesDataSet.write((const void*)posesData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL); free((void*)posesData); posesDataSet.close(); posesDS.close(); // Creating points dataset const hsize_t pointsChunkDim[] = {10, 3}; H5::DSetCreatPropList pointsPropList; pointsPropList.setLayout(H5D_CHUNKED); pointsPropList.setChunk(2, pointsChunkDim); pointsPropList.setDeflate(9); const hsize_t pointsMaxDim[] = { H5S_UNLIMITED, 3 }; const hsize_t pointsCurDim[] = { 0, 3 }; H5::DataSpace pointsDS(2, pointsCurDim, pointsMaxDim); H5::DataSet pointsDataSet = geometryGroup.createDataSet("Points", H5::PredType::IEEE_F64LE, pointsDS, pointsPropList); pointsDataSet.close(); pointsDS.close(); // Creating inliers dataset const hsize_t inliersChunkDim[] = { 3 }; H5::DSetCreatPropList inliersPropList; inliersPropList.setLayout(H5D_CHUNKED); inliersPropList.setChunk(1, inliersChunkDim); inliersPropList.setDeflate(9); const hsize_t inliersMaxDim[] = { H5S_UNLIMITED }; const hsize_t inliersCurDim[] = { frames_.size() }; H5::DataSpace inliersDS(1, inliersCurDim, inliersMaxDim); H5::VarLenType inliersType(&H5::PredType::STD_U8LE); H5::DataSet inliersDataSet = geometryGroup.createDataSet("Inliers", inliersType, inliersDS, inliersPropList); inliersDataSet.close(); inliersType.close(); inliersDS.close(); // Creating curve dataset const hsize_t chunkDim[] = { 5 }; H5::DSetCreatPropList propList; propList.setLayout(H5D_CHUNKED); propList.setChunk(1, chunkDim); propList.setDeflate(9); H5::VarLenType curveDatasetType(&H5::PredType::STD_U64LE); hsize_t curvesDim[] = { 0 }; hsize_t curvesMaxDim[] = { H5S_UNLIMITED }; H5::DataSpace curvesDataspace(1, curvesDim, curvesMaxDim); H5::DataSet curvesDataset = geometryGroup.createDataSet("Curves", curveDatasetType, curvesDataspace, propList); curvesDataset.close(); curvesDataspace.close(); curveDatasetType.close(); propList.close(); geometryGroup.close(); }
void Bundle2::saveGeometry(const boost::filesystem::path& fileName) const { H5::H5File bundleFile; bundleFile.openFile(fileName.string(), H5F_ACC_RDWR); H5::Group rootGroup = bundleFile.openGroup("/"); // If the group "Geometry" exists, delete it! if(checkGeometry_(bundleFile)) { rootGroup.unlink("Geometry"); } // Creating group Geometry H5::Group geometryGroup = rootGroup.createGroup("Geometry"); // Saving poses const hsize_t posesChunkDim[] = { 3, 12 }; H5::DSetCreatPropList posesPropList; posesPropList.setLayout(H5D_CHUNKED); posesPropList.setChunk(2, posesChunkDim); posesPropList.setDeflate(9); const hsize_t posesMaxDim[] = { H5S_UNLIMITED, 12 }; const hsize_t posesCurDim[] = { frames_.size(), 12 }; H5::DataSpace posesDS(2, posesCurDim, posesMaxDim); H5::DataSet posesDataSet = geometryGroup.createDataSet("Poses", H5::PredType::IEEE_F64LE, posesDS, posesPropList); double* posesData = (double*)malloc(frames_.size()*12*sizeof(double)); size_t i = 0; for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) { posesData[i*12] = (*it)->pose()->t().x(); posesData[i*12 + 1] = (*it)->pose()->t().y(); posesData[i*12 + 2] = (*it)->pose()->t().z(); core::Matrix<double> R = (*it)->pose()->R(); posesData[i*12 + 3] = R[0][0]; posesData[i*12 + 4] = R[1][0]; posesData[i*12 + 5] = R[2][0]; posesData[i*12 + 6] = R[0][1]; posesData[i*12 + 7] = R[1][1]; posesData[i*12 + 8] = R[2][1]; posesData[i*12 + 9] = R[0][2]; posesData[i*12 + 10] = R[1][2]; posesData[i*12 + 11] = R[2][2]; ++i; } posesDataSet.write((const void*)posesData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL); free((void*)posesData); posesDataSet.close(); posesDS.close(); // Saving points const hsize_t pointsChunkDim[] = {10, 3}; H5::DSetCreatPropList pointsPropList; pointsPropList.setLayout(H5D_CHUNKED); pointsPropList.setChunk(2, pointsChunkDim); pointsPropList.setDeflate(9); const hsize_t pointsMaxDim[] = { H5S_UNLIMITED, 3 }; const hsize_t pointsCurDim[] = { tracks_.size(), 3 }; H5::DataSpace pointsDS(2, pointsCurDim, pointsMaxDim); H5::DataSet pointsDataSet = geometryGroup.createDataSet("Points", H5::PredType::IEEE_F64LE, pointsDS, pointsPropList); double* pointsData = (double*)malloc(tracks_.size()*3*sizeof(double)); i = 0; for(deque<Track*>::const_iterator it = tracks_.begin(); it != tracks_.end(); it++) { pointsData[i*3] = (*it)->point()->coords().x(); pointsData[i*3 + 1] = (*it)->point()->coords().y(); pointsData[i*3 + 2] = (*it)->point()->coords().z(); ++i; } pointsDataSet.write((const void*)pointsData, H5::PredType::NATIVE_DOUBLE, H5::DataSpace::ALL, H5::DataSpace::ALL); free((void*)pointsData); pointsDataSet.close(); pointsDS.close(); // Saving inlier information const hsize_t inliersChunkDim[] = { 3 }; H5::DSetCreatPropList inliersPropList; inliersPropList.setLayout(H5D_CHUNKED); inliersPropList.setChunk(1, inliersChunkDim); inliersPropList.setDeflate(9); const hsize_t inliersMaxDim[] = { H5S_UNLIMITED }; const hsize_t inliersCurDim[] = { frames_.size() }; H5::DataSpace inliersDS(1, inliersCurDim, inliersMaxDim); H5::VarLenType inliersType(&H5::PredType::STD_U8LE); H5::DataSet inliersDataSet = geometryGroup.createDataSet("Inliers", inliersType, inliersDS, inliersPropList); i = 0; for(deque<Frame*>::const_iterator it = frames_.begin(); it != frames_.end(); it++) { hvl_t inliersLine; size_t inliersLineSize = 0; for(size_t j = 0; j < (*it)->size(); ++j) { View& v = (**it)[j]; for(unsigned int cam = 0; cam < v.numCameras(); ++cam) { if(v.inCamera(cam)) ++inliersLineSize; } } inliersLine.len = inliersLineSize; inliersLine.p = malloc(inliersLineSize*sizeof(unsigned char)); size_t k = 0; for(size_t j = 0; j < (*it)->size(); ++j) { View& v = (**it)[j]; for(unsigned int cam = 0; cam < v.numCameras(); ++cam) { if(v.inCamera(cam)) { ((unsigned char*)(inliersLine.p))[k] = v.ray(cam).inlier()?1:0; ++k; } } } const hsize_t dsOffset[] = { i }; const hsize_t dsCount[] = { 1 }; H5::DataSpace inliersCurDS = inliersDataSet.getSpace(); inliersCurDS.selectHyperslab(H5S_SELECT_SET, dsCount, dsOffset); const hsize_t memDim[] = { 1 }; H5::DataSpace memDS(1, memDim, memDim); H5::VarLenType memType(&H5::PredType::NATIVE_UCHAR); inliersDataSet.write((const void*)&inliersLine, memType, memDS, inliersCurDS); memType.close(); memDS.close(); inliersCurDS.close(); free(inliersLine.p); ++i; } inliersDataSet.close(); inliersType.close(); inliersDS.close(); // Saving curves if(!curves_.empty()) { const hsize_t chunkDim[] = { 5 }; H5::DSetCreatPropList propList; propList.setLayout(H5D_CHUNKED); propList.setChunk(1, chunkDim); propList.setDeflate(9); H5::VarLenType curveDatasetType(&H5::PredType::STD_U64LE); hsize_t curvesDim[] = { curves_.size() }; hsize_t curvesMaxDim[] = { H5S_UNLIMITED }; H5::DataSpace curvesDataspace(1, curvesDim, curvesMaxDim); H5::DataSet curvesDataset = geometryGroup.createDataSet("Curves", curveDatasetType, curvesDataspace, propList); for(size_t i = 0; i < curves_.size(); ++i) { hvl_t curveLine; curveLine.len = curves_[i].size(); curveLine.p = malloc(curves_[i].size()*sizeof(size_t)); for(size_t j = 0; j < curves_[i].size(); ++j) ((size_t*)(curveLine.p))[j] = curves_[i].track(j); const hsize_t dsOffset[] = { i }; const hsize_t dsCount[] = { 1 }; H5::DataSpace curDS = curvesDataset.getSpace(); curDS.selectHyperslab(H5S_SELECT_SET, dsCount, dsOffset); const hsize_t memDim[] = { 1 }; H5::DataSpace memDS(1, memDim, memDim); H5::VarLenType memType(&H5::PredType::NATIVE_HSIZE); curvesDataset.write((const void*)&curveLine, memType, memDS, curDS); memType.close(); memDS.close(); curDS.close(); free(curveLine.p); } curvesDataset.close(); curvesDataspace.close(); curveDatasetType.close(); propList.close(); } geometryGroup.close(); rootGroup.close(); bundleFile.close(); }