Tuplef Wrapper_i_hdf::get_dims()const { Tuplef tmp; H5File file = H5File( file_name_, H5F_ACC_RDONLY ); Group group = Group(file.openGroup("/")); Attr_list_hdf attr_list(&group); if(!attr_list.contains_attr("version")) throw invalid_argument("input file does not have a version"); int file_version = 0; if(attr_list.get_value("version",file_version) != 1) throw invalid_argument("input file is not the right version"); if(attr_list.contains_attr("dims")) { attr_list.get_value("dims",tmp); } group.close(); file.close(); return Tuplef(tmp); }
//'@title Function to write a matrix chunk to file //' //'@description Function is intended for internal use //' //'@param dset character denoting the meta data name of the data set //'@param chunk matrix that will be written to h5file //'@param dim numeric containing the dimension of the matrix that will be written to file //'@param filePath character denoting the location of the h5 file //'@return int 0 // [[Rcpp::export]] int h5WriteDoubleMat (std::string dset, SEXP chunk, NumericVector dim, std::string filePath) { H5File *file = new H5File(filePath, H5F_ACC_RDWR); // Data initialization. int rank = 2; hsize_t dims[rank]; // dataset dimensions for(int k = 0; k < rank; k++) dims[k] = dim(k); const void *buf = REAL(chunk); // Create the data space for the dataset. DataSpace dataspace (rank, dims, NULL); // Create the dataset. H5std_string dsetName(dset); DataSet dataset = file->createDataSet(dsetName, PredType::NATIVE_DOUBLE, dataspace); // Write the data to the dataset using default memory space, file // space, and transfer properties. dataset.write(buf, PredType::NATIVE_DOUBLE); dataset.close(); //nn file->close(); return 0; }
// Function to return a selected part of a data frame as a list List ch5ChunkSel(string chunkName, CharacterVector selCols, string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); // converting the R object to a numeric matrix NumericMatrix M = as<NumericMatrix>(data); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); // Create the output List DF; string colName; string colClass; NumericVector vec; CharacterVector levels; int n = selCols.size(); IntegerVector sel(n); int selN; // First we need to find which of the columns has been selected sel = match(selCols, colNames); for(int i = 0; i < n; i++) { colName = selCols[i]; selN = sel[i] - 1; colClass = colClasses[selN]; if(colClass != "factor") { DF[colName] = M(_, selN); }else{ vec = M(_, selN); levels = (CharacterVector)ch5ReadFactor(colName, filePath); DF[colName] = cCreateFactor(vec, levels); } } dataset.close(); file->close(); return DF; }
//'@title Function for dummy read //' //'@param chunkName the name of the chunk to be read back //'@param filePath the path to the h5 file //'@return int 0 // [[Rcpp::export]] int h5DummyRead(std::string chunkName, std::string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); dataset.close(); file->close(); return 0; }
CharacterVector ch5ReadFactor(string charName, string filePath) { H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Group Meta Group Group* metaGroup = new Group(file->openGroup("/MetaData/Factor")); // Getting the data set from the file DataSet dataset = metaGroup->openDataSet((H5std_string)charName); // Getting the data space from the dataset DataSpace dataspace = dataset.getSpace(); // We know that it is a char vector array so ndim = 1 hsize_t dims[1]; // Getting the length of strings dataspace.getSimpleExtentDims(dims, NULL); // for convenience int dim = dims[0]; // String Type StrType vlst(0, H5T_VARIABLE); // Returning the data char *strRet[dim]; dataset.read(strRet, vlst); CharacterVector out(dim); for(int i = 0; i < dim; i++) { out[i] = strRet[i]; } dataset.close(); //nn metaGroup->close(); file->close(); return out; }
//'@title This function writes an integer meta data to file //' //'@description This function is inteded for internal use //' //'@param intName the name of the meta data item to be written //'@param integer int that will be written to the meta data described by intName //'@param filePath character path to the h5 file where data will be written //'@param update int flag for whether item is new (0) or whether it will overwrite a previous item (1) //'@return int 0 // [[Rcpp::export]] int h5WriteInt(std::string intName, int integer, std::string filePath, int update) { H5File *file = new H5File(filePath, H5F_ACC_RDWR); // Colclasses dim hsize_t dim[1] = {1}; string meta = "/MetaData"; // Group Meta Group Group* metaGroup = new Group(file->openGroup(meta)); // dataspace DataSpace dataspace = DataSpace(1, dim); DataSet dataset; if(update == 1) { string slash = "/"; string groupName = meta + slash + intName; file->unlink(groupName); } dataset = metaGroup->createDataSet(intName, PredType::NATIVE_INT, dataspace); dataset.write(&integer, PredType::NATIVE_INT); dataset.close(); //nn metaGroup->close(); file->close(); return 0; }
//'@title Function creates the groups for the meta data to be written to the h5 file //' //'@description Function to create the groups in the h5 file before it is populated. //'This function is intended for internal use only //' //'@param filePath character path to the location where the h5 file will be written //'@return int 0 // [[Rcpp::export]] int h5CreateMetaData(std::string filePath) { H5File *file = new H5File(filePath, H5F_ACC_RDWR); hsize_t dims[1] = {1}; // The variable length string type StrType vlst(0, H5T_VARIABLE); // Creating the meta data group Group *metaGroup = new Group(file->createGroup("/MetaData")); // File path H5std_string fString("FilePath"); DataSpace fileDataSpace (1, dims, NULL); DataSet fileDataSet; // Create a dataset in the group fileDataSet = metaGroup->createDataSet(fString, vlst, fileDataSpace); fileDataSet.write(filePath, vlst); // Create the factor group Group *factorGroup = new Group(metaGroup->createGroup("/MetaData/Factor")); fileDataSet.close(); //nn factorGroup->close(); metaGroup->close(); file->close(); return 0; }
//'@title Legacy function to return a data frame chunk as a list //' //'@description Experimental function not intended for use at all //' //'@param chunkName the name of the chunk to be read //'@param filePath the path to the h5 file //'@return List of the data frame chunk // [[Rcpp::export]] SEXP h5ReadDoubleMat3(std::string chunkName, std::string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace //double (*buf)[dims[1]]*[dims[0]] = malloc(dims[1]]*[dims[0] * sizeof *buf); //buf[dims[1]][dims[0]] = 0.0; double **buf = (double**) calloc (dims[1]*dims[0], sizeof(double)); buf[dims[1]][dims[0]] = 0.0; //double buf[dims[1]][dims[0]]; dataset.read(buf, PredType::NATIVE_DOUBLE, dataspace); // Attempt tp append the contents to a list List out; NumericVector vec(dims[0]); NumericMatrix M(dims[0], dims[1]); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); string colName; for(int i = 0; i < dims[1]; i++) { NumericVector vec(dims[0]); for(int j = 0; j < dims[0]; j++) { M(j,i) = buf[i][j]; vec(j) = buf[i][j]; } colName = colNames[i]; if(colClasses[i] == "factor") { CharacterVector levels; levels = h5ReadFactor(colName, filePath); IntegerVector fact(vec.size()); fact = cCreateFactor(vec, levels); out[colName] = fact; }else{ out[colName] = vec; } } free(buf); dataset.close(); //nn file->close(); return wrap(out); }
//'@title Legacy function to return a data frame chunk as a list //' //'@description Experimental function not intended for use at all //' //'@param chunkName the name of the chunk to be read //'@param filePath the path to the h5 file //'@return List of the data frame chunk // [[Rcpp::export]] SEXP h5ReadDoubleMat2(std::string chunkName, std::string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); // converting the R object to a numeric matrix NumericMatrix M = as<NumericMatrix>(data); List out; NumericVector vec(dims[0]); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); string colName; for(int i = 0; i < dims[1]; i++) { NumericVector vec(dims[0]); for(int j = 0; j < dims[0]; j++) { vec(j) = M(j,i); } colName = colNames[i]; if(colClasses[i] == "factor") { CharacterVector levels; levels = ch5ReadFactor(colName, filePath); IntegerVector fact(vec.size()); fact = cCreateFactor(vec, levels); out[colName] = fact; }else{ out[colName] = vec; } } dataset.close(); //nn file->close(); // Returning the data return wrap(out); }
int main(int argc, char ** argv) { if (argc < 3){ cout << "Usage:" << argv[0] << " <filename> <outputfilename> - display some statistics of specified synapse." << endl; return 0; } const H5std_string FILE_NAME(argv[1]); ofstream outfile; outfile.open(argv[2]); try{ H5File * file = new H5File(FILE_NAME, H5F_ACC_RDONLY); Group * netgroup = new Group(file->openGroup("network")); const DataSet * syndataset = new DataSet(netgroup->openDataSet("synapse")); synstat(*syndataset, outfile); file->close(); outfile.close(); } catch( FileIException error ) { error.printError(); return -1; } // catch failure caused by the DataSet operations catch( DataSetIException error ) { error.printError(); return -1; } // catch failure caused by the DataSpace operations catch( DataSpaceIException error ) { error.printError(); return -1; } // catch failure caused by the DataSpace operations catch( DataTypeIException error ) { error.printError(); return -1; } return 0; }
//'@title Function to create file a h5 file //' //'@description Function to create a h5 file. It is intended for internal use only //' //'@param filePath a character denoting the path to the location where the h5 file will be written //'@param overwrite integer 1 for overwrite and 0 for not overwrite. Will fail if overwrite is 0 //'@return int 0 // [[Rcpp::export]] int h5CreateFile(std::string filePath, int overwrite) { H5File* file; if(fileExists(filePath)) { if(overwrite) { file = new H5File(filePath, H5F_ACC_TRUNC); }else{ throw "Error: file exists and overwrite is set to 0."; } }else{ file = new H5File(filePath, H5F_ACC_TRUNC); } file->close(); return 0; }
//'@title Function to read an integer item from meta data //' //'@param intName character for the name of the item to be read back //'@param filePath character for the path to the h5 file //'@return int iteger item defined by intName in the meta data // [[Rcpp::export]] int h5ReadInt(std::string intName, std::string filePath) { H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Group Meta Group Group* metaGroup = new Group(file->openGroup("/MetaData")); // Getting the data set from the file DataSet dataset = metaGroup->openDataSet((H5std_string)intName); // Getting the data space from the dataset DataSpace dataspace = dataset.getSpace(); // Returning the data int intRet; dataset.read(&intRet, PredType::NATIVE_INT); dataset.close(); //nn metaGroup->close(); file->close(); return intRet; }
//'@title This function writes a character vector to the meta data //' //'@description This function writes a character vector to the meta data and is intended for internal use. //' //'@param charName the name that will be given to the meta data character vector //'@param charVec the character vector to be written as meta data //'@param filePath the path to the h5 file where the data will be written //'@param update integer denoting whether the data item is new or whether it is an update //'(which will overwrite any previous item) //'@return int 0 // [[Rcpp::export]] int h5WriteCharVector(std::string charName, SEXP charVec, std::string filePath, int update) { H5File *file = new H5File(filePath, H5F_ACC_RDWR); int len = Rf_length(charVec); hsize_t DIM1 = len; int rank = 1; //cout << "The length is ... " << len << endl; // Create a datatype to refer to StrType vlst(0, H5T_VARIABLE); // This is the char array char** arr = convertCharArray(charVec); string meta = "/MetaData"; // Group Meta Group Group* metaGroup = new Group(file->openGroup(meta)); // The dataset and dataspace hsize_t dims[] = {DIM1}; //hsize_t maxdims[] = {H5S_UNLIMITED}; DataSet dataset; if(update == 1) { string slash = "/"; string groupName = meta + slash + charName; file->unlink(groupName); } DataSpace dataspace(rank, dims); dataset = metaGroup->createDataSet(charName, vlst, dataspace); dataset.write(arr, vlst); dataset.close(); //nn metaGroup->close(); file->close(); return 0; }
bool Wrapper_i_hdf::priv_init(int fr_count) { if(locked_) return false; try { H5File * file = new H5File( file_name_, H5F_ACC_RDONLY ); if(two_d_data_) { const string nop_str = "number-of-planes"; Group g = file->openGroup("/"); Attr_list_hdf atr_list(&g); if(!atr_list.contains_attr(nop_str)) throw logic_error("wrapper_i_hdf: number-of-planes not found in file"); atr_list.get_value(nop_str,frame_count_); } else { /** @todo deal with this better, don't have any data */ frame_count_ = 1; } if(fr_count != 0) { if(fr_count + start_ > frame_count_) throw runtime_error("wrapper_i_hdf: asking for too many frames"); frame_count_ = fr_count; } // logic to set up data maps and data storage int i_count =0; int f_count =0; int c_count =0; for(set<pair<D_TYPE,int> >::iterator it = data_types_.begin(); it != data_types_.end();++it) { D_TYPE cur = (*it).first; switch(v_type(cur)) { case V_INT: data_i_.push_back(vector<int*>(frame_count_)); d_mapi_.set_lookup(cur,i_count++); break; case V_FLOAT: data_f_.push_back(vector<float*>(frame_count_)); d_mapf_.set_lookup(cur,f_count++); break; case V_COMPLEX: data_c_.push_back(vector<complex<float>*>(frame_count_)); d_mapc_.set_lookup(cur,c_count++); break; case V_STRING: case V_BOOL: case V_GUID: case V_TIME: case V_UINT: case V_ERROR: throw logic_error("wrapper_i_hdf: The data type should not have been " + VT2str_s(v_type(cur))); } } frame_c_.reserve(frame_count_); if(two_d_data_) frame_zdata_.resize(frame_count_); // set the size of the md_store set_Md_store_size(frame_count_); // fill in data // assume that the frames run from 0 to frame_count_ for(unsigned int j = 0; j<frame_count_;++j) { string frame_name = format_name(j+start_); Group * frame = new Group(file->openGroup(frame_name)); Attr_list_hdf g_attr_list(frame); set_Md_store(j,new Md_store(g_attr_list)); if(two_d_data_) { if(!g_attr_list.contains_attr("z-position")) throw logic_error("wrapper_i_hdf: z-position not found"); g_attr_list.get_value("z-position",frame_zdata_[j]); } for(set<pair<D_TYPE,int> >::iterator it = data_types_.begin(); it != data_types_.end();++it) { if(two_d_data_ && ((*it).first)==utilities::D_ZPOS) continue; // *************** DataSet * dset = new DataSet(frame->openDataSet(format_dset_name((*it).first,(*it).second))); // *************** DataSpace dspace = dset-> getSpace(); dspace.selectAll(); int part_count = dspace.getSimpleExtentNpoints(); // if the first data set for this frame set the number of particles if(frame_c_.size()==j) frame_c_.push_back(part_count); // if the part_count is less than a previous dataset, set the // number of particles to be the smaller number. This // shouldn't result in memory leaks as the bare arrays are // never returned else if(frame_c_.at(j) > part_count) frame_c_.at(j) = part_count; // if the current set has more than a previous set, keep the // old value. these checks are a kludge, need to deal with // this better at the level of writing out the data else if(frame_c_.at(j) < part_count) continue; // if(frame_c_.at(j) != part_count) // throw runtime_error("wrapper_i_hdf: data sets different sizes"); D_TYPE cur_type = (*it).first; switch(v_type(cur_type)) { case V_INT: data_i_.at(d_mapi_(cur_type)).at(j) = new int [part_count]; dset->read(data_i_.at(d_mapi_(cur_type)).at(j),PredType::NATIVE_INT); break; case V_FLOAT: data_f_.at(d_mapf_(cur_type)).at(j) = new float [part_count]; dset->read(data_f_.at(d_mapf_(cur_type)).at(j),PredType::NATIVE_FLOAT); break; case V_COMPLEX: throw logic_error("not implemented yet"); break; case V_STRING: case V_BOOL: case V_GUID: case V_TIME: case V_UINT: case V_ERROR: throw logic_error("wrapper_i_hdf: The data type should not have been " + VT2str_s(v_type(cur_type))); } // clean up hdf stuff dset->close(); delete dset; dset = NULL; } frame->close(); delete frame; frame = NULL; } file->close(); delete file; file = NULL; // shift all of the z by the minimum to start at zero if(two_d_data_) { float min = frame_zdata_[0]; for(unsigned int j = 0; j<frame_count_;++j) if(frame_zdata_[j]<min) min = frame_zdata_[j]; for(unsigned int j = 0; j<frame_count_;++j) frame_zdata_[j] -= min ; } } catch(Exception & e) { // clean up data if it died e.printError(); throw runtime_error("wrapper_i_hdf: constructor error"); } for(unsigned int j= 0; j<frame_count_;++j) total_part_count_ += frame_c_.at(j); return true; }
//' @title Fast model frame for activeReg //' //' @description Function returns a scaled down model frame essentially returning list with no NA values. //' Each item in the list represents a column in the data frame. //' //' @param chunkName character name of the chunk to be read //' @param selCols character vector of columns to select //' @param filePath character path to file where chunk is to be read from //' @return list representing a data frame with no NA values. //[[Rcpp::export]] SEXP h5ModelFrame(std::string chunkName, SEXP selCols_, std::string filePath) { // Quick conversion of the SEXP column selection to character vector CharacterVector selCols(selCols_); // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); // Convert the R object to a numeric matrix NumericMatrix M__(data); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); // Create the output List DF; string colName; string colClass; NumericVector vect; CharacterVector levels; int n = selCols.size(); IntegerVector sel(n); int selN; NumericMatrix M_(M__.nrow(), n); // Find which of the columns has been selected sel = match(selCols, colNames); // Copy the correct matrix columns for(int i = 0; i < n; i++) { selN = sel[i] - 1; M_(_, i) = M__(_, selN); } // Number of rows in the matrix int nr = M_.nrow(); int goodRow; NumericVector goodRows(nr); int badRow; for(int i = 0; i < nr; i++) { badRow = sum(is_na(M_(i, _))); if(badRow >= 1) { goodRows[i] = 0; }else{ goodRows[i] = 1; } } //goodRows = goodRows*-1 + 1; NumericMatrix M(sum(goodRows), n); int j = 0; // Remove NA rows for(int i = 0; i < nr; i++) { goodRow = goodRows[i]; if(goodRow == 1) { M(j, _) = M_(i, _); j++; } } // Compile the list for(int i = 0; i < n; i++) { colName = selCols[i]; selN = sel[i] - 1; colClass = colClasses[selN]; if(colClass != "factor") { DF[colName] = M(_, i); }else{ vect = M(_, i); levels = (CharacterVector)ch5ReadFactor(colName, filePath); DF[colName] = cCreateFactor(vect, levels); } } dataset.close(); file->close(); return wrap(DF); }
//'@title Function to close the h5 file //' //'@description Closes the h5 file //' //'@param filePath character path to the file which will be flushed //'@return int 0 // [[Rcpp::export]] int h5CloseFile(std::string filePath) { H5File *file = new H5File(filePath, H5F_ACC_RDWR); file->close(); return 0; }