int TestCompress() { unsigned int flags = 0; unsigned int config = 0; size_t cd_nelemts = 0; TESTING("compression") #ifdef H5_HAVE_FILTER_DEFLATE try { /* Create packet table with compression. */ FL_PacketTable wrapper(fileID, "/compressTest", H5T_NATIVE_CHAR, 100, 8); /* Create an HDF5 C++ file object */ H5File file; file.setId(fileID); /* Make sure that the deflate filter is set by opening the packet table * as a dataset and getting its creation property list */ DataSet dsetID = file.openDataSet("/compressTest"); DSetCreatPropList dcplID = dsetID.getCreatePlist(); dcplID.getFilterById(H5Z_FILTER_DEFLATE, flags, cd_nelemts, NULL, 0, NULL, config); } catch (Exception e) { H5_FAILED(); return 1; } PASSED(); #else SKIPPED(); puts(" deflate filter not enabled"); #endif /* H5_HAVE_FILTER_DEFLATE */ return 0; }
//'@title Function for dummy read //' //'@param chunkName the name of the chunk to be read back //'@param filePath the path to the h5 file //'@return int 0 // [[Rcpp::export]] int h5DummyRead(std::string chunkName, std::string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); dataset.close(); file->close(); return 0; }
// Function to return a selected part of a data frame as a list List ch5ChunkSel(string chunkName, CharacterVector selCols, string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); // converting the R object to a numeric matrix NumericMatrix M = as<NumericMatrix>(data); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); // Create the output List DF; string colName; string colClass; NumericVector vec; CharacterVector levels; int n = selCols.size(); IntegerVector sel(n); int selN; // First we need to find which of the columns has been selected sel = match(selCols, colNames); for(int i = 0; i < n; i++) { colName = selCols[i]; selN = sel[i] - 1; colClass = colClasses[selN]; if(colClass != "factor") { DF[colName] = M(_, selN); }else{ vec = M(_, selN); levels = (CharacterVector)ch5ReadFactor(colName, filePath); DF[colName] = cCreateFactor(vec, levels); } } dataset.close(); file->close(); return DF; }
//'@title Legacy function to return a data frame chunk as a list //' //'@description Experimental function not intended for use at all //' //'@param chunkName the name of the chunk to be read //'@param filePath the path to the h5 file //'@return List of the data frame chunk // [[Rcpp::export]] SEXP h5ReadDoubleMat3(std::string chunkName, std::string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace //double (*buf)[dims[1]]*[dims[0]] = malloc(dims[1]]*[dims[0] * sizeof *buf); //buf[dims[1]][dims[0]] = 0.0; double **buf = (double**) calloc (dims[1]*dims[0], sizeof(double)); buf[dims[1]][dims[0]] = 0.0; //double buf[dims[1]][dims[0]]; dataset.read(buf, PredType::NATIVE_DOUBLE, dataspace); // Attempt tp append the contents to a list List out; NumericVector vec(dims[0]); NumericMatrix M(dims[0], dims[1]); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); string colName; for(int i = 0; i < dims[1]; i++) { NumericVector vec(dims[0]); for(int j = 0; j < dims[0]; j++) { M(j,i) = buf[i][j]; vec(j) = buf[i][j]; } colName = colNames[i]; if(colClasses[i] == "factor") { CharacterVector levels; levels = h5ReadFactor(colName, filePath); IntegerVector fact(vec.size()); fact = cCreateFactor(vec, levels); out[colName] = fact; }else{ out[colName] = vec; } } free(buf); dataset.close(); //nn file->close(); return wrap(out); }
void read_feature_size(H5File h5f, Size &size_out, const char *name) { DataSet dataset = h5f.openDataSet(name); DataSpace dspace = dataset.getSpace(); assert (dspace.getSimpleExtentNdims() == 2); hsize_t dims[2]; dspace.getSimpleExtentDims(dims); size_out.height = dims[0]; size_out.width = dims[1]; }
//'@title Legacy function to return a data frame chunk as a list //' //'@description Experimental function not intended for use at all //' //'@param chunkName the name of the chunk to be read //'@param filePath the path to the h5 file //'@return List of the data frame chunk // [[Rcpp::export]] SEXP h5ReadDoubleMat2(std::string chunkName, std::string filePath) { // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); // converting the R object to a numeric matrix NumericMatrix M = as<NumericMatrix>(data); List out; NumericVector vec(dims[0]); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); string colName; for(int i = 0; i < dims[1]; i++) { NumericVector vec(dims[0]); for(int j = 0; j < dims[0]; j++) { vec(j) = M(j,i); } colName = colNames[i]; if(colClasses[i] == "factor") { CharacterVector levels; levels = ch5ReadFactor(colName, filePath); IntegerVector fact(vec.size()); fact = cCreateFactor(vec, levels); out[colName] = fact; }else{ out[colName] = vec; } } dataset.close(); //nn file->close(); // Returning the data return wrap(out); }
Scalar readScalar(H5File &file, string DSitem){ DataSet item = file.openDataSet(DSitem); DataSpace dsp = item.getSpace(); // assert(dsp.getSimpleExtentNdims() == 0); // hsize_t dims[1]; // int ndims = dsp.getSimpleExtentDims(dims, NULL); Scalar value(0); PredType type = getPredType<Scalar>(); item.read(&value, type); return value; }
void read_hdf5_image(H5File h5f, Mat &image_out, const char *name, const Rect &roi=Rect(0,0,0,0)) { DataSet dataset = h5f.openDataSet(name); DataSpace dspace = dataset.getSpace(); assert (dspace.getSimpleExtentNdims() == 2); hsize_t dims[2]; dspace.getSimpleExtentDims(dims); if ((roi.width == 0) && (roi.height == 0)) { image_out.create(dims[0], dims[1], CV_32F); dspace.selectAll(); } else { image_out.create(roi.height, roi.width, CV_32F); hsize_t _offset[2], _size[2]; _offset[0] = roi.y; _offset[1] = roi.x; _size[0] = roi.height; _size[1] = roi.width; dspace.selectHyperslab(H5S_SELECT_SET, _size, _offset); } DataSpace imspace; float *imdata; if (image_out.isContinuous()) { dims[0] = image_out.size().height; dims[1] = image_out.size().width; imspace = DataSpace(2, dims); imspace.selectAll(); imdata = image_out.ptr<float>(); } else { // we are working with an ROI assert (image_out.isSubmatrix()); Size parent_size; Point parent_ofs; image_out.locateROI(parent_size, parent_ofs); hsize_t parent_count[2]; parent_count[0] = parent_size.height; parent_count[1] = parent_size.width; imspace.setExtentSimple(2, parent_count); hsize_t im_offset[2], im_size[2]; im_offset[0] = parent_ofs.y; im_offset[1] = parent_ofs.x; im_size[0] = image_out.size().height; im_size[1] = image_out.size().width; imspace.selectHyperslab(H5S_SELECT_SET, im_size, im_offset); imdata = image_out.ptr<float>() - parent_ofs.x - parent_ofs.y * parent_size.width; } dataset.read(imdata, PredType::NATIVE_FLOAT, imspace, dspace); }
/* Helper routine for test_vl_rewrite() */ static void read_scalar_dset(H5File& file, DataType& type, DataSpace& space, char *name, char *data) { char *data_read; DataSet dset; try { dset = file.openDataSet(name); dset.read(&data_read, type, space, space); dset.close(); if(HDstrcmp(data, data_read)) TestErrPrintf("Expected %s for dataset %s but read %s\n", data, name, data_read); HDfree(data_read); } // end try catch (FileIException ferr) { throw; } catch (DataSetIException derr) { throw; } }
int main(void) { /* First structure and dataset*/ typedef struct s1_t { int a; float b; double c; } s1_t; /* Second structure (subset of s1_t) and dataset*/ typedef struct s2_t { double c; int a; } s2_t; // Try block to detect exceptions raised by any of the calls inside it try { /* * Initialize the data */ int i; s1_t s1[LENGTH]; for (i = 0; i< LENGTH; i++) { s1[i].a = i; s1[i].b = i*i; s1[i].c = 1./(i+1); } /* * Turn off the auto-printing when failure occurs so that we can * handle the errors appropriately */ Exception::dontPrint(); /* * Create the data space. */ hsize_t dim[] = {LENGTH}; /* Dataspace dimensions */ DataSpace space( RANK, dim ); /* * Create the file. */ H5File* file = new H5File( FILE_NAME, H5F_ACC_TRUNC ); /* * Create the memory datatype. */ CompType mtype1( sizeof(s1_t) ); mtype1.insertMember( MEMBER1, HOFFSET(s1_t, a), PredType::NATIVE_INT); mtype1.insertMember( MEMBER3, HOFFSET(s1_t, c), PredType::NATIVE_DOUBLE); mtype1.insertMember( MEMBER2, HOFFSET(s1_t, b), PredType::NATIVE_FLOAT); /* * Create the dataset. */ DataSet* dataset; dataset = new DataSet(file->createDataSet(DATASET_NAME, mtype1, space)); /* * Write data to the dataset; */ dataset->write( s1, mtype1 ); /* * Release resources */ delete dataset; delete file; /* * Open the file and the dataset. */ file = new H5File( FILE_NAME, H5F_ACC_RDONLY ); dataset = new DataSet (file->openDataSet( DATASET_NAME )); /* * Create a datatype for s2 */ CompType mtype2( sizeof(s2_t) ); mtype2.insertMember( MEMBER3, HOFFSET(s2_t, c), PredType::NATIVE_DOUBLE); mtype2.insertMember( MEMBER1, HOFFSET(s2_t, a), PredType::NATIVE_INT); /* * Read two fields c and a from s1 dataset. Fields in the file * are found by their names "c_name" and "a_name". */ s2_t s2[LENGTH]; dataset->read( s2, mtype2 ); /* * Display the fields */ cout << endl << "Field c : " << endl; for( i = 0; i < LENGTH; i++) cout << s2[i].c << " "; cout << endl; cout << endl << "Field a : " << endl; for( i = 0; i < LENGTH; i++) cout << s2[i].a << " "; cout << endl; /* * Create a datatype for s3. */ CompType mtype3( sizeof(float) ); mtype3.insertMember( MEMBER2, 0, PredType::NATIVE_FLOAT); /* * Read field b from s1 dataset. Field in the file is found by its name. */ float s3[LENGTH]; // Third "structure" - used to read float field of s1 dataset->read( s3, mtype3 ); /* * Display the field */ cout << endl << "Field b : " << endl; for( i = 0; i < LENGTH; i++) cout << s3[i] << " "; cout << endl; /* * Release resources */ delete dataset; delete file; } // end of try block // catch failure caused by the H5File operations catch( FileIException error ) { error.printErrorStack(); return -1; } // catch failure caused by the DataSet operations catch( DataSetIException error ) { error.printErrorStack(); return -1; } // catch failure caused by the DataSpace operations catch( DataSpaceIException error ) { error.printErrorStack(); return -1; } // catch failure caused by the DataSpace operations catch( DataTypeIException error ) { error.printErrorStack(); return -1; } return 0; }
/*------------------------------------------------------------------------- * Function: test_create * * Purpose: Attempts to create a dataset. * * Return: Success: 0 * * Failure: -1 * * Programmer: Binh-Minh Ribler (using C version) * Friday, January 5, 2001 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t test_create( H5File& file) { SUBTEST("create, open, close"); // Setting this to NULL for cleaning up in failure situations DataSet *dataset = NULL; try { // Create a data space hsize_t dims[2]; dims[0] = 256; dims[1] = 512; DataSpace space (2, dims, NULL); // Create a dataset using the default dataset creation properties. // We're not sure what they are, so we won't check. dataset = new DataSet (file.createDataSet (DSET_DEFAULT_NAME, PredType::NATIVE_DOUBLE, space)); // Add a comment to the dataset file.setComment (DSET_DEFAULT_NAME, "This is a dataset"); // Close the dataset delete dataset; dataset = NULL; // Try creating a dataset that already exists. This should fail since a // dataset can only be created once. If an exception is not thrown for // this action by createDataSet, then throw an invalid action exception. try { dataset = new DataSet (file.createDataSet (DSET_DEFAULT_NAME, PredType::NATIVE_DOUBLE, space)); // continuation here, that means no exception has been thrown throw InvalidActionException("H5File::createDataSet", "Library allowed overwrite of existing dataset"); } catch (FileIException E) // catching invalid creating dataset {} // do nothing, exception expected // Open the dataset we created above and then close it. This is one // way to open an existing dataset for accessing. dataset = new DataSet (file.openDataSet (DSET_DEFAULT_NAME)); // Get and verify the name of this dataset, using // H5std_string getObjName() H5std_string ds_name = dataset->getObjName(); verify_val(ds_name, DSET_DEFAULT_NAME_PATH, "DataSet::getObjName", __LINE__, __FILE__); // Get and verify the comment from this dataset, using // H5std_string getComment(const H5std_string& name, <buf_size=0, by default>) H5std_string comment = file.getComment(DSET_DEFAULT_NAME); verify_val(comment, "This is a dataset", "DataSet::getComment", __LINE__, __FILE__); // Close the dataset when accessing is completed delete dataset; // This is another way to open an existing dataset for accessing. DataSet another_dataset(file.openDataSet (DSET_DEFAULT_NAME)); // Try opening a non-existent dataset. This should fail so if an // exception is not thrown for this action by openDataSet, then // display failure information and throw an exception. try { dataset = new DataSet (file.openDataSet( "does_not_exist" )); // continuation here, that means no exception has been thrown throw InvalidActionException("H5File::openDataSet", "Attempted to open a non-existent dataset"); } catch (FileIException E ) // catching creating non-existent dataset {} // do nothing, exception expected // Create a new dataset that uses chunked storage instead of the default // layout. DSetCreatPropList create_parms; hsize_t csize[2]; csize[0] = 5; csize[1] = 100; create_parms.setChunk( 2, csize ); dataset = new DataSet (file.createDataSet (DSET_CHUNKED_NAME, PredType::NATIVE_DOUBLE, space, create_parms)); // Note: this one has no error message in C when failure occurs? // clean up and return with success delete dataset; PASSED(); return 0; } // outer most try block catch (InvalidActionException E) { cerr << " FAILED" << endl; cerr << " <<< " << E.getDetailMsg() << " >>>" << endl << endl; // clean up and return with failure if (dataset != NULL) delete dataset; return -1; } // catch all other exceptions catch (Exception E) { issue_fail_msg("test_create", __LINE__, __FILE__); // clean up and return with failure if (dataset != NULL) delete dataset; return -1; } } // test_create
/*------------------------------------------------------------------------- * Function: test_multiopen * * Purpose: Tests that a bug no longer exists. If a dataset is opened * twice and one of the handles is used to extend the dataset, * then the other handle should return the new size when * queried. * * Return: Success: 0 * * Failure: -1 * * Programmer: Binh-Minh Ribler (using C version) * Saturday, February 17, 2001 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t test_multiopen (H5File& file) { SUBTEST("Multi-open with extending"); DataSpace* space = NULL; try { // Create a dataset creation property list DSetCreatPropList dcpl; // Set chunk size to given size hsize_t cur_size[1] = {10}; dcpl.setChunk (1, cur_size); // Create a simple data space with unlimited size static hsize_t max_size[1] = {H5S_UNLIMITED}; space = new DataSpace (1, cur_size, max_size); // Create first dataset DataSet dset1 = file.createDataSet ("multiopen", PredType::NATIVE_INT, *space, dcpl); // Open again the first dataset from the file to another DataSet object. DataSet dset2 = file.openDataSet ("multiopen"); // Relieve the dataspace delete space; space = NULL; // Extend the dimensionality of the first dataset cur_size[0] = 20; dset1.extend (cur_size); // Get the size from the second handle space = new DataSpace (dset2.getSpace()); hsize_t tmp_size[1]; space->getSimpleExtentDims (tmp_size); if (cur_size[0]!=tmp_size[0]) { cerr << " Got " << (int)tmp_size[0] << " instead of " << (int)cur_size[0] << "!" << endl; throw Exception("test_multiopen", "Failed in multi-open with extending"); } // clean up and return with success delete space; PASSED(); return 0; } // end try block // catch all dataset, file, space, and plist exceptions catch (Exception E) { cerr << " FAILED" << endl; cerr << " <<< " << E.getDetailMsg() << " >>>" << endl << endl; // clean up and return with failure if (space != NULL) delete space; return -1; } } // test_multiopen
/*------------------------------------------------------------------------- * Function: test_compression * * Purpose: Tests dataset compression. If compression is requested when * it hasn't been compiled into the library (such as when * updating an existing compressed dataset) then data is sent to * the file uncompressed but no errors are returned. * * Return: Success: 0 * * Failure: -1 * * Programmer: Binh-Minh Ribler (using C version) * Friday, January 5, 2001 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t test_compression(H5File& file) { #ifndef H5_HAVE_FILTER_DEFLATE const char *not_supported; not_supported = " Deflate compression is not enabled."; #endif /* H5_HAVE_FILTER_DEFLATE */ int points[100][200]; int check[100][200]; hsize_t i, j, n; // Initialize the dataset for (i = n = 0; i < 100; i++) { for (j = 0; j < 200; j++) { points[i][j] = (int)n++; } } char* tconv_buf = new char [1000]; DataSet* dataset = NULL; try { const hsize_t size[2] = {100, 200}; // Create the data space DataSpace space1(2, size, NULL); // Create a small conversion buffer to test strip mining DSetMemXferPropList xfer; xfer.setBuffer (1000, tconv_buf, NULL); // Use chunked storage with compression DSetCreatPropList dscreatplist; const hsize_t chunk_size[2] = {2, 25}; dscreatplist.setChunk (2, chunk_size); dscreatplist.setDeflate (6); #ifdef H5_HAVE_FILTER_DEFLATE SUBTEST("Compression (setup)"); // Create the dataset dataset = new DataSet (file.createDataSet (DSET_COMPRESS_NAME, PredType::NATIVE_INT, space1, dscreatplist)); PASSED(); /*---------------------------------------------------------------------- * STEP 1: Read uninitialized data. It should be zero. *---------------------------------------------------------------------- */ SUBTEST("Compression (uninitialized read)"); dataset->read ((void*) check, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); for (i=0; i<size[0]; i++) { for (j=0; j<size[1]; j++) { if (0!=check[i][j]) { H5_FAILED(); cerr << " Read a non-zero value." << endl; cerr << " At index " << (unsigned long)i << "," << (unsigned long)j << endl; throw Exception("test_compression", "Failed in uninitialized read"); } } } PASSED(); /*---------------------------------------------------------------------- * STEP 2: Test compression by setting up a chunked dataset and writing * to it. *---------------------------------------------------------------------- */ SUBTEST("Compression (write)"); for (i=n=0; i<size[0]; i++) { for (j=0; j<size[1]; j++) { points[i][j] = (int)n++; } } dataset->write ((void*) points, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); PASSED(); /*---------------------------------------------------------------------- * STEP 3: Try to read the data we just wrote. *---------------------------------------------------------------------- */ SUBTEST("Compression (read)"); // Read the dataset back dataset->read ((void*)check, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); // Check that the values read are the same as the values written for (i = 0; i < size[0]; i++) for (j = 0; j < size[1]; j++) { int status = check_values (i, j, points[i][j], check[i][j]); if (status == -1) throw Exception("test_compression", "Failed in read"); } PASSED(); /*---------------------------------------------------------------------- * STEP 4: Write new data over the top of the old data. The new data is * random thus not very compressible, and will cause the chunks to move * around as they grow. We only change values for the left half of the * dataset although we rewrite the whole thing. *---------------------------------------------------------------------- */ SUBTEST("Compression (modify)"); for (i=0; i<size[0]; i++) { for (j=0; j<size[1]/2; j++) { points[i][j] = rand (); } } dataset->write ((void*)points, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); // Read the dataset back and check it dataset->read ((void*)check, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); // Check that the values read are the same as the values written for (i = 0; i < size[0]; i++) for (j = 0; j < size[1]; j++) { int status = check_values (i, j, points[i][j], check[i][j]); if (status == -1) throw Exception("test_compression", "Failed in modify"); } PASSED(); /*---------------------------------------------------------------------- * STEP 5: Close the dataset and then open it and read it again. This * insures that the compression message is picked up properly from the * object header. *---------------------------------------------------------------------- */ SUBTEST("Compression (re-open)"); // close this dataset to reuse the var delete dataset; dataset = new DataSet (file.openDataSet (DSET_COMPRESS_NAME)); dataset->read ((void*)check, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); // Check that the values read are the same as the values written for (i = 0; i < size[0]; i++) for (j = 0; j < size[1]; j++) { int status = check_values (i, j, points[i][j], check[i][j]); if (status == -1) throw Exception("test_compression", "Failed in re-open"); } PASSED(); /*---------------------------------------------------------------------- * STEP 6: Test partial I/O by writing to and then reading from a * hyperslab of the dataset. The hyperslab does not line up on chunk * boundaries (we know that case already works from above tests). *---------------------------------------------------------------------- */ SUBTEST("Compression (partial I/O)"); const hsize_t hs_size[2] = {4, 50}; const hsize_t hs_offset[2] = {7, 30}; for (i = 0; i < hs_size[0]; i++) { for (j = 0; j < hs_size[1]; j++) { points[hs_offset[0]+i][hs_offset[1]+j] = rand (); } } space1.selectHyperslab( H5S_SELECT_SET, hs_size, hs_offset ); dataset->write ((void*)points, PredType::NATIVE_INT, space1, space1, xfer); dataset->read ((void*)check, PredType::NATIVE_INT, space1, space1, xfer); // Check that the values read are the same as the values written for (i=0; i<hs_size[0]; i++) { for (j=0; j<hs_size[1]; j++) { if (points[hs_offset[0]+i][hs_offset[1]+j] != check[hs_offset[0]+i][hs_offset[1]+j]) { H5_FAILED(); cerr << " Read different values than written.\n" << endl; cerr << " At index " << (unsigned long)(hs_offset[0]+i) << "," << (unsigned long)(hs_offset[1]+j) << endl; cerr << " At original: " << (int)points[hs_offset[0]+i][hs_offset[1]+j] << endl; cerr << " At returned: " << (int)check[hs_offset[0]+i][hs_offset[1]+j] << endl; throw Exception("test_compression", "Failed in partial I/O"); } } // for j } // for i delete dataset; dataset = NULL; PASSED(); #else SUBTEST("deflate filter"); SKIPPED(); cerr << not_supported << endl; #endif /*---------------------------------------------------------------------- * STEP 7: Register an application-defined compression method and use it * to write and then read the dataset. *---------------------------------------------------------------------- */ SUBTEST("Compression (app-defined method)"); if (H5Zregister (H5Z_BOGUS)<0) throw Exception("test_compression", "Failed in app-defined method"); if (H5Pset_filter (dscreatplist.getId(), H5Z_FILTER_BOGUS, 0, 0, NULL)<0) throw Exception("test_compression", "Failed in app-defined method"); dscreatplist.setFilter (H5Z_FILTER_BOGUS, 0, 0, NULL); DataSpace space2 (2, size, NULL); dataset = new DataSet (file.createDataSet (DSET_BOGUS_NAME, PredType::NATIVE_INT, space2, dscreatplist)); dataset->write ((void*)points, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); dataset->read ((void*)check, PredType::NATIVE_INT, DataSpace::ALL, DataSpace::ALL, xfer); // Check that the values read are the same as the values written for (i = 0; i < size[0]; i++) for (j = 0; j < size[1]; j++) { int status = check_values (i, j, points[i][j], check[i][j]); if (status == -1) throw Exception("test_compression", "Failed in app-defined method"); } PASSED(); /*---------------------------------------------------------------------- * Cleanup *---------------------------------------------------------------------- */ delete dataset; delete [] tconv_buf; return 0; } // end try // catch all dataset, file, space, and plist exceptions catch (Exception E) { cerr << " FAILED" << endl; cerr << " <<< " << E.getDetailMsg() << " >>>" << endl << endl; // clean up and return with failure if (dataset != NULL) delete dataset; if (tconv_buf) delete [] tconv_buf; return -1; } } // test_compression
//' @title Fast model frame for activeReg //' //' @description Function returns a scaled down model frame essentially returning list with no NA values. //' Each item in the list represents a column in the data frame. //' //' @param chunkName character name of the chunk to be read //' @param selCols character vector of columns to select //' @param filePath character path to file where chunk is to be read from //' @return list representing a data frame with no NA values. //[[Rcpp::export]] SEXP h5ModelFrame(std::string chunkName, SEXP selCols_, std::string filePath) { // Quick conversion of the SEXP column selection to character vector CharacterVector selCols(selCols_); // Open the file in Read/Write Mode, H5F_ACC_RDONLY H5File *file = new H5File(filePath, H5F_ACC_RDONLY); // Opening the data set DataSet dataset = file->openDataSet((H5std_string)chunkName); // Opening the data space DataSpace dataspace = dataset.getSpace(); // Get the number of dimensions int ndim = dataspace.getSimpleExtentNdims(); // Create a dimension object to be filled with the dimensions of the data set hsize_t dims[ndim]; // Fill the dimension of the dataset dataspace.getSimpleExtentDims(dims, NULL); // Create the return data // Filling the matrix with data form the dataspace SEXP data; // Allocating a matrix of the right size and dimension data = PROTECT(Rf_allocMatrix(REALSXP, dims[0], dims[1])); // Filling the matrix with data form the dataspace dataset.read(REAL(data), PredType::NATIVE_DOUBLE, dataspace); UNPROTECT(1); // Convert the R object to a numeric matrix NumericMatrix M__(data); CharacterVector colNames = ch5ReadCharVector("ColumnNames", filePath); CharacterVector colClasses = ch5ReadCharVector("ColumnClasses", filePath); // Create the output List DF; string colName; string colClass; NumericVector vect; CharacterVector levels; int n = selCols.size(); IntegerVector sel(n); int selN; NumericMatrix M_(M__.nrow(), n); // Find which of the columns has been selected sel = match(selCols, colNames); // Copy the correct matrix columns for(int i = 0; i < n; i++) { selN = sel[i] - 1; M_(_, i) = M__(_, selN); } // Number of rows in the matrix int nr = M_.nrow(); int goodRow; NumericVector goodRows(nr); int badRow; for(int i = 0; i < nr; i++) { badRow = sum(is_na(M_(i, _))); if(badRow >= 1) { goodRows[i] = 0; }else{ goodRows[i] = 1; } } //goodRows = goodRows*-1 + 1; NumericMatrix M(sum(goodRows), n); int j = 0; // Remove NA rows for(int i = 0; i < nr; i++) { goodRow = goodRows[i]; if(goodRow == 1) { M(j, _) = M_(i, _); j++; } } // Compile the list for(int i = 0; i < n; i++) { colName = selCols[i]; selN = sel[i] - 1; colClass = colClasses[selN]; if(colClass != "factor") { DF[colName] = M(_, i); }else{ vect = M(_, i); levels = (CharacterVector)ch5ReadFactor(colName, filePath); DF[colName] = cCreateFactor(vect, levels); } } dataset.close(); file->close(); return wrap(DF); }
int main (void) { int i,j; // loop indices */ /* * Try block to detect exceptions raised by any of the calls inside it */ try { /* * Turn off the auto-printing when failure occurs so that we can * handle the errors appropriately */ Exception::dontPrint(); /* * Create a file. */ H5File* file = new H5File( FILE_NAME, H5F_ACC_TRUNC ); /* * Create property list for a dataset and set up fill values. */ int fillvalue = 0; /* Fill value for the dataset */ DSetCreatPropList plist; plist.setFillValue(PredType::NATIVE_INT, &fillvalue); /* * Create dataspace for the dataset in the file. */ hsize_t fdim[] = {FSPACE_DIM1, FSPACE_DIM2}; // dim sizes of ds (on disk) DataSpace fspace( FSPACE_RANK, fdim ); /* * Create dataset and write it into the file. */ DataSet* dataset = new DataSet(file->createDataSet( DATASET_NAME, PredType::NATIVE_INT, fspace, plist)); /* * Select hyperslab for the dataset in the file, using 3x2 blocks, * (4,3) stride and (2,4) count starting at the position (0,1). */ hsize_t start[2]; // Start of hyperslab hsize_t stride[2]; // Stride of hyperslab hsize_t count[2]; // Block count hsize_t block[2]; // Block sizes start[0] = 0; start[1] = 1; stride[0] = 4; stride[1] = 3; count[0] = 2; count[1] = 4; block[0] = 3; block[1] = 2; fspace.selectHyperslab( H5S_SELECT_SET, count, start, stride, block); /* * Create dataspace for the first dataset. */ hsize_t dim1[] = {MSPACE1_DIM}; /* Dimension size of the first dataset (in memory) */ DataSpace mspace1( MSPACE1_RANK, dim1 ); /* * Select hyperslab. * We will use 48 elements of the vector buffer starting at the * second element. Selected elements are 1 2 3 . . . 48 */ start[0] = 1; stride[0] = 1; count[0] = 48; block[0] = 1; mspace1.selectHyperslab( H5S_SELECT_SET, count, start, stride, block); /* * Write selection from the vector buffer to the dataset in the file. * * File dataset should look like this: * 0 1 2 0 3 4 0 5 6 0 7 8 * 0 9 10 0 11 12 0 13 14 0 15 16 * 0 17 18 0 19 20 0 21 22 0 23 24 * 0 0 0 0 0 0 0 0 0 0 0 0 * 0 25 26 0 27 28 0 29 30 0 31 32 * 0 33 34 0 35 36 0 37 38 0 39 40 * 0 41 42 0 43 44 0 45 46 0 47 48 * 0 0 0 0 0 0 0 0 0 0 0 0 */ int vector[MSPACE1_DIM]; // vector buffer for dset /* * Buffer initialization. */ vector[0] = vector[MSPACE1_DIM - 1] = -1; for (i = 1; i < MSPACE1_DIM - 1; i++) vector[i] = i; dataset->write( vector, PredType::NATIVE_INT, mspace1, fspace ); /* * Reset the selection for the file dataspace fid. */ fspace.selectNone(); /* * Create dataspace for the second dataset. */ hsize_t dim2[] = {MSPACE2_DIM}; /* Dimension size of the second dataset (in memory */ DataSpace mspace2( MSPACE2_RANK, dim2 ); /* * Select sequence of NPOINTS points in the file dataspace. */ hsize_t coord[NPOINTS][FSPACE_RANK]; /* Array to store selected points from the file dataspace */ coord[0][0] = 0; coord[0][1] = 0; coord[1][0] = 3; coord[1][1] = 3; coord[2][0] = 3; coord[2][1] = 5; coord[3][0] = 5; coord[3][1] = 6; fspace.selectElements( H5S_SELECT_SET, NPOINTS, (const hsize_t *)coord); /* * Write new selection of points to the dataset. */ int values[] = {53, 59, 61, 67}; /* New values to be written */ dataset->write( values, PredType::NATIVE_INT, mspace2, fspace ); /* * File dataset should look like this: * 53 1 2 0 3 4 0 5 6 0 7 8 * 0 9 10 0 11 12 0 13 14 0 15 16 * 0 17 18 0 19 20 0 21 22 0 23 24 * 0 0 0 59 0 61 0 0 0 0 0 0 * 0 25 26 0 27 28 0 29 30 0 31 32 * 0 33 34 0 35 36 67 37 38 0 39 40 * 0 41 42 0 43 44 0 45 46 0 47 48 * 0 0 0 0 0 0 0 0 0 0 0 0 * */ /* * Close the dataset and the file. */ delete dataset; delete file; /* * Open the file. */ file = new H5File( FILE_NAME, H5F_ACC_RDONLY ); /* * Open the dataset. */ dataset = new DataSet( file->openDataSet( DATASET_NAME )); /* * Get dataspace of the dataset. */ fspace = dataset->getSpace(); /* * Select first hyperslab for the dataset in the file. The following * elements are selected: * 10 0 11 12 * 18 0 19 20 * 0 59 0 61 * */ start[0] = 1; start[1] = 2; block[0] = 1; block[1] = 1; stride[0] = 1; stride[1] = 1; count[0] = 3; count[1] = 4; fspace.selectHyperslab(H5S_SELECT_SET, count, start, stride, block); /* * Add second selected hyperslab to the selection. * The following elements are selected: * 19 20 0 21 22 * 0 61 0 0 0 * 27 28 0 29 30 * 35 36 67 37 38 * 43 44 0 45 46 * 0 0 0 0 0 * Note that two hyperslabs overlap. Common elements are: * 19 20 * 0 61 */ start[0] = 2; start[1] = 4; block[0] = 1; block[1] = 1; stride[0] = 1; stride[1] = 1; count[0] = 6; count[1] = 5; fspace.selectHyperslab(H5S_SELECT_OR, count, start, stride, block); /* * Create memory dataspace. */ hsize_t mdim[] = {MSPACE_DIM1, MSPACE_DIM2}; /* Dimension sizes of the dataset in memory when we read selection from the dataset on the disk */ DataSpace mspace(MSPACE_RANK, mdim); /* * Select two hyperslabs in memory. Hyperslabs has the same * size and shape as the selected hyperslabs for the file dataspace. */ start[0] = 0; start[1] = 0; block[0] = 1; block[1] = 1; stride[0] = 1; stride[1] = 1; count[0] = 3; count[1] = 4; mspace.selectHyperslab(H5S_SELECT_SET, count, start, stride, block); start[0] = 1; start[1] = 2; block[0] = 1; block[1] = 1; stride[0] = 1; stride[1] = 1; count[0] = 6; count[1] = 5; mspace.selectHyperslab(H5S_SELECT_OR, count, start, stride, block); /* * Initialize data buffer. */ int matrix_out[MSPACE_DIM1][MSPACE_DIM2]; for (i = 0; i < MSPACE_DIM1; i++) for (j = 0; j < MSPACE_DIM2; j++) matrix_out[i][j] = 0; /* * Read data back to the buffer matrix. */ dataset->read(matrix_out, PredType::NATIVE_INT, mspace, fspace); /* * Display the result. Memory dataset is: * * 10 0 11 12 0 0 0 0 0 * 18 0 19 20 0 21 22 0 0 * 0 59 0 61 0 0 0 0 0 * 0 0 27 28 0 29 30 0 0 * 0 0 35 36 67 37 38 0 0 * 0 0 43 44 0 45 46 0 0 * 0 0 0 0 0 0 0 0 0 * 0 0 0 0 0 0 0 0 0 */ for (i=0; i < MSPACE_DIM1; i++) { for(j=0; j < MSPACE_DIM2; j++) cout << matrix_out[i][j] << " "; cout << endl; } /* * Close the dataset and the file. */ delete dataset; delete file; } // end of try block // catch failure caused by the H5File operations catch( FileIException error ) { error.printErrorStack(); return -1; } // catch failure caused by the DataSet operations catch( DataSetIException error ) { error.printErrorStack(); return -1; } // catch failure caused by the DataSpace operations catch( DataSpaceIException error ) { error.printErrorStack(); return -1; } return 0; }