sz_params* H5Z_SZ_Init_Default() { herr_t ret = H5Zregister(H5Z_SZ); sz_params* conf_params = (sz_params *)malloc(sizeof(sz_params)); confparams_cpr->quantization_intervals = 0; confparams_cpr->max_quant_intervals = 65536; dataEndianType = LITTLE_ENDIAN_DATA; confparams_cpr->sol_ID = SZ; //confparams_cpr->layers = 1; confparams_cpr->sampleDistance = 100; confparams_cpr->predThreshold = 0.99; //confparams_cpr->offset = 0; confparams_cpr->szMode = SZ_BEST_COMPRESSION; confparams_cpr->gzipMode = 1; //best speed confparams_cpr->errorBoundMode = REL; //details about errorBoundMode can be found in sz.config confparams_cpr->absErrBound = 1E-4; confparams_cpr->relBoundRatio = 1E-3; confparams_cpr->pw_relBoundRatio = 1E-4; confparams_cpr->segment_size = 32; confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE; int status = SZ_Init_Params(conf_params); if(status == SZ_NSCS || ret < 0) return NULL; else return confparams_cpr; }
int H5Z_SZ_Init_Params(sz_params *params) { herr_t ret = H5Zregister(H5Z_SZ); int status = SZ_Init_Params(params); if(status == SZ_NSCS || ret < 0) return SZ_NSCS; else return SZ_SCES; }
/*------------------------------------------------------------------------- * Function: create_file_with_bogus_filter * * Purpose: Create a file and a dataset with a bogus filter enabled * * Return: Success: 0 * * Failure: -1 * * Programmer: Raymond Lu * 2 June 2011 * *------------------------------------------------------------------------- */ static herr_t create_file_with_bogus_filter(void) { hid_t fid = -1; /* file ID */ hid_t dsid = -1; /* dataset ID */ hid_t sid = -1; /* dataspace ID */ hid_t dcpl = -1; /* dataset creation property list ID */ hsize_t dims[1] = {20}; /* dataspace dimensions */ hsize_t chunk_dims[1] = {10}; /* chunk dimensions */ int buf[20]; int rank = 1; int i; for(i = 0; i < 20; i++) buf[i] = 1; /* create a file using default properties */ if((fid = H5Fcreate(TESTFILE2, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; /* create a data space */ if((sid = H5Screate_simple(rank, dims, NULL)) < 0) goto error; /* create dcpl */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error; /* create chunking */ if(H5Pset_chunk(dcpl, rank, chunk_dims) < 0) goto error; /* register bogus filter */ if(H5Zregister (H5Z_BOGUS) < 0) goto error; if(H5Pset_filter(dcpl, H5Z_FILTER_BOGUS, 0, (size_t)0, NULL) < 0) goto error; /* create a dataset */ if((dsid = H5Dcreate2(fid, DSETNAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; if(H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) goto error; /* close */ if(H5Pclose(dcpl) < 0) goto error; if(H5Dclose(dsid) < 0) goto error; if(H5Sclose(sid) < 0) goto error; if(H5Fclose(fid) < 0) goto error; return 0; error: H5E_BEGIN_TRY { H5Pclose(dcpl); H5Dclose(dsid); H5Sclose(sid); H5Fclose(fid); } H5E_END_TRY; return -1; }
int register_lzo(char **version, char **date) { #ifdef HAVE_LZO_LIB /* The conditional below is somewhat messy, but it is necessary because the THG team has decided to fix an API inconsistency in the definition of the H5Z_class_t structure in version 1.8.3 */ #if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 7) || \ (H5_USE_16_API && (H5_VERS_MAJOR > 1 || \ (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR > 8 || \ (H5_VERS_MINOR == 8 && H5_VERS_RELEASE >= 3))))) /* 1.6.x */ H5Z_class_t filter_class = { (H5Z_filter_t)(FILTER_LZO), /* filter_id */ "lzo", /* comment */ NULL, /* can_apply_func */ NULL, /* set_local_func */ (H5Z_func_t)(lzo_deflate) /* filter_func */ }; #else /* 1.8.x where x < 3 */ H5Z_class_t filter_class = { H5Z_CLASS_T_VERS, /* H5Z_class_t version */ (H5Z_filter_t)(FILTER_LZO), /* filter_id */ 1, 1, /* Encoding and decoding enabled */ "lzo", /* comment */ NULL, /* can_apply_func */ NULL, /* set_local_func */ (H5Z_func_t)(lzo_deflate) /* filter_func */ }; #endif /* Init the LZO library */ if (lzo_init()!=LZO_E_OK) { fprintf(stderr, "Problems initializing LZO library\n"); *version = NULL; *date = NULL; return 0; /* lib is not available */ } /* Register the lzo compressor */ H5Zregister(&filter_class); *version = strdup(LZO_VERSION_STRING); *date = strdup(LZO_VERSION_DATE); return 1; /* lib is available */ #else *version = NULL; *date = NULL; return 0; /* lib is not available */ #endif /* HAVE_LZO_LIB */ }
/*------------------------------------------------------------------------- * Function: create_dataset * * Purpose: Creates a square dataset with square chunks, registers a * stupid compress/uncompress pair for counting I/O, and * initializes the dataset. The chunk size is in bytes, the * dataset size is in terms of chunks. * * Return: void * * Programmer: Robb Matzke * Thursday, May 14, 1998 * * Modifications: * *------------------------------------------------------------------------- */ static void create_dataset (void) { hid_t file, space, dcpl, dset; hsize_t size[2]; signed char *buf; /* The file */ file = H5Fcreate (FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_g); /* The data space */ size[0] = size[1] = DS_SIZE * CH_SIZE; space = H5Screate_simple (2, size, size); /* The storage layout and compression */ dcpl = H5Pcreate (H5P_DATASET_CREATE); size[0] = size[1] = CH_SIZE; H5Pset_chunk (dcpl, 2, size); #ifdef H5_WANT_H5_V1_4_COMPAT H5Zregister (FILTER_COUNTER, "counter", counter); #else /* H5_WANT_H5_V1_4_COMPAT */ H5Zregister (H5Z_COUNTER); #endif /* H5_WANT_H5_V1_4_COMPAT */ H5Pset_filter (dcpl, FILTER_COUNTER, 0, 0, NULL); /* The dataset */ dset = H5Dcreate (file, "dset", H5T_NATIVE_SCHAR, space, dcpl); assert (dset>=0); /* The data */ buf = calloc (1, SQUARE (DS_SIZE*CH_SIZE)); H5Dwrite (dset, H5T_NATIVE_SCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); free (buf); /* Close */ H5Dclose (dset); H5Sclose (space); H5Pclose (dcpl); H5Fclose (file); }
/*------------------------------------------------------------------------- * Function: test_filter_write_failure * * Purpose: Tests the library's behavior when a mandate filter returns * failure. There're only 5 chunks with each of them having * 2 integers. The filter will fail in the last chunk. The * dataset should release all resources even though the last * chunk can't be flushed to file. The file should close * successfully. * * Return: * Success: 0 * Failure: -1 * * Programmer: Raymond Lu * 25 August 2010 * * Modifications: * Raymond Lu * 5 Oct 2010 * Test when the chunk cache is enable and disabled to make * sure the library behaves properly. *------------------------------------------------------------------------- */ static herr_t test_filter_write(char *file_name, hid_t my_fapl, hbool_t cache_enabled) { hid_t file = -1; hid_t dataset=-1; /* dataset ID */ hid_t sid=-1; /* dataspace ID */ hid_t dcpl=-1; /* dataset creation property list ID */ hsize_t dims[1]={DIM}; /* dataspace dimension - 10*/ hsize_t chunk_dims[1]={FILTER_CHUNK_DIM}; /* chunk dimension - 2*/ int points[DIM]; /* Data */ herr_t ret; /* generic return value */ int i; if(cache_enabled) { TESTING("data writing when a mandatory filter fails and chunk cache is enabled"); } else { TESTING("data writing when a mandatory filter fails and chunk cache is disabled"); } /* Create file */ if((file = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) TEST_ERROR /* create the data space */ if((sid = H5Screate_simple(1, dims, NULL)) < 0) TEST_ERROR /* Create dcpl and register the filter */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR if(H5Pset_chunk(dcpl, 1, chunk_dims) < 0) TEST_ERROR if(H5Zregister (H5Z_FAIL_TEST) < 0) TEST_ERROR /* Check that the filter was registered */ if(TRUE != H5Zfilter_avail(H5Z_FILTER_FAIL_TEST)) FAIL_STACK_ERROR /* Enable the filter as mandatory */ if(H5Pset_filter(dcpl, H5Z_FILTER_FAIL_TEST, 0, (size_t)0, NULL) < 0) TEST_ERROR /* create a dataset */ if((dataset = H5Dcreate2(file, DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) TEST_ERROR /* Initialize the write buffer */ for(i = 0; i < DIM; i++) points[i] = i; /* Write data. If the chunk cache is enabled, H5Dwrite should succeed. If it is * diabled, H5Dwrite should fail. */ if(cache_enabled) { if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, sid, H5P_DEFAULT, points) < 0) TEST_ERROR } else {
loaded_hdf5_plugin() { int reg_rval = 0; if (!H5Zfilter_avail(H5Z_FILTER_SQY)) reg_rval = H5Zregister(H5Z_SQY); if (reg_rval < 0) std::cerr << __FILE__ << ":"<< __LINE__ << "\t unable to register sqy as hdf5 filter!!\n"; else std::cout << "Done registering sqy as hdf5 filter.\n"; }
int H5Z_SZ_Init(char* cfgFile) { herr_t ret; //printf("start in H5Z_SZ_Init, load_conffile_flag = %d\n", load_conffile_flag); if(load_conffile_flag==0) { load_conffile_flag = 1; int status = SZ_Init(cfgFile); //printf("cfgFile=%s\n", cfgFile); //printf("szMode=%d, errorBoundMode=%d, relBoundRatio=%f\n", szMode, errorBoundMode, relBoundRatio); if(status == SZ_NSCS) return SZ_NSCS; else return SZ_SCES; } ret = H5Zregister(H5Z_SZ); if(ret < 0) return SZ_NSCS; else return SZ_SCES; }
int register_bzip2(char **version, char **date) { #ifdef HAVE_BZ2_LIB char *libver, *versionp, *datep, *sep; H5Z_class_t filter_class = { H5Z_CLASS_T_VERS, /* H5Z_class_t version */ (H5Z_filter_t)(FILTER_BZIP2), /* filter_id */ 1, 1, /* Encoding and decoding enabled */ "bzip2", /* comment */ NULL, /* can_apply_func */ NULL, /* set_local_func */ (H5Z_func_t)(bzip2_deflate) /* filter_func */ }; /* Register the filter class for the bzip2 compressor. */ H5Zregister(&filter_class); /* Get the library major version from the version string. */ libver = strdup(BZ2_bzlibVersion()); sep = strchr(libver, ','); assert(sep != NULL); assert(*(sep + 1) == ' '); *sep = '\0'; versionp = libver; datep = sep + 2; /* after the comma and a space */ *version = strdup(versionp); *date = strdup(datep); free(libver); return 1; /* library is available */ #else return 0; /* library is not available */ #endif /* defined HAVE_BZ2_LIB */ }
int register_lzo(char **version, char **date) { #ifdef HAVE_LZO_LIB H5Z_class_t filter_class = { H5Z_CLASS_T_VERS, /* H5Z_class_t version */ (H5Z_filter_t)(FILTER_LZO), /* filter_id */ 1, 1, /* Encoding and decoding enabled */ "lzo", /* comment */ NULL, /* can_apply_func */ NULL, /* set_local_func */ (H5Z_func_t)(lzo_deflate) /* filter_func */ }; /* Init the LZO library */ if (lzo_init()!=LZO_E_OK) { fprintf(stderr, "Problems initializing LZO library\n"); *version = NULL; *date = NULL; return 0; /* lib is not available */ } /* Register the lzo compressor */ H5Zregister(&filter_class); *version = strdup(LZO_VERSION_STRING); *date = strdup(LZO_VERSION_DATE); return 1; /* lib is available */ #else *version = NULL; *date = NULL; return 0; /* lib is not available */ #endif /* HAVE_LZO_LIB */ }
/*------------------------------------------------------------------------- * Function: test_skip_compress_write2 * * Purpose: Test skipping compression filter when there are three filters * for the dataset * * Return: Success: 0 * * Failure: 1 * * Programmer: Raymond Lu * 30 November 2012 * *------------------------------------------------------------------------- */ static int test_skip_compress_write2(hid_t file) { hid_t dataspace = -1, dataset = -1; hid_t mem_space = -1; hid_t cparms = -1, dxpl = -1; hsize_t dims[2] = {NX, NY}; hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; herr_t status; int i, j, n; unsigned filter_mask = 0; int origin_direct_buf[CHUNK_NX][CHUNK_NY]; int direct_buf[CHUNK_NX][CHUNK_NY]; int check_chunk[CHUNK_NX][CHUNK_NY]; hsize_t offset[2] = {0, 0}; size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); int aggression = 9; /* Compression aggression setting */ hsize_t start[2]; /* Start of hyperslab */ hsize_t stride[2]; /* Stride of hyperslab */ hsize_t count[2]; /* Block count */ hsize_t block[2]; /* Block sizes */ TESTING("skipping compression filters but keep two other filters"); /* * Create the data space with unlimited dimensions. */ if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) goto error; if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) goto error; /* * Modify dataset creation properties, i.e. enable chunking and compression. * The order of filters is bogus 1 + deflate + bogus 2. */ if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error; if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) goto error; /* Register and enable first bogus filter */ if(H5Zregister (H5Z_BOGUS1) < 0) goto error; if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS1, 0, (size_t)0, NULL) < 0) goto error; /* Enable compression filter */ if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) goto error; /* Register and enable second bogus filter */ if(H5Zregister (H5Z_BOGUS2) < 0) goto error; if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS2, 0, (size_t)0, NULL) < 0) goto error; /* * Create a new dataset within the file using cparms * creation properties. */ if((dataset = H5Dcreate2(file, DATASETNAME3, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) goto error; if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) goto error; /* Initialize data for one chunk. Apply operations of two bogus filters to the chunk */ for(i = n = 0; i < CHUNK_NX; i++) for(j = 0; j < CHUNK_NY; j++) { origin_direct_buf[i][j] = n++; direct_buf[i][j] = (origin_direct_buf[i][j] + ADD_ON) * FACTOR; } /* write the uncompressed chunk data repeatedly to dataset, using the direct writing function. * Indicate skipping the compression filter but keep the other two bogus filters */ offset[0] = CHUNK_NX; offset[1] = CHUNK_NY; /* compression filter is the middle one to be skipped */ filter_mask = 0x00000002; if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) goto error; if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) goto error; if(H5Dclose(dataset) < 0) goto error; if((dataset = H5Dopen2(file, DATASETNAME3, H5P_DEFAULT)) < 0) goto error; /* * Select hyperslab for one chunk in the file */ start[0] = CHUNK_NX; start[1] = CHUNK_NY; stride[0] = 1; stride[1] = 1; count[0] = 1; count[1] = 1; block[0] = CHUNK_NX; block[1] = CHUNK_NY; if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) goto error; /* Read the chunk back */ if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) goto error; /* Check that the values read are the same as the values written */ for(i = 0; i < CHUNK_NX; i++) { for(j = 0; j < CHUNK_NY; j++) { if(origin_direct_buf[i][j] != check_chunk[i][j]) { printf(" 1. Read different values than written."); printf(" At index %d,%d\n", i, j); printf(" origin_direct_buf=%d, check_chunk=%d\n", origin_direct_buf[i][j], check_chunk[i][j]); goto error; } } } /* * Close/release resources. */ H5Dclose(dataset); H5Sclose(mem_space); H5Sclose(dataspace); H5Pclose(cparms); H5Pclose(dxpl); PASSED(); return 0; error: H5E_BEGIN_TRY { H5Dclose(dataset); H5Sclose(mem_space); H5Sclose(dataspace); H5Pclose(cparms); H5Pclose(dxpl); } H5E_END_TRY; return 1; }
void cbf_register_filters() { H5Zregister(&H5Z_LZ4); H5Zregister(&bshuf_H5Filter); }
/*------------------------------------------------------------------------- * Function: test_filter_write_failure * * Purpose: Tests the library's behavior when a mandate filter returns * failure. There're only 5 chunks with each of them having * 2 integers. The filter will fail in the last chunk. The * dataset should release all resources even though the last * chunk can't be flushed to file. The file should close * successfully. * * Return: * Success: 0 * Failure: -1 * * Programmer: Raymond Lu * 25 August 2010 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t test_filter_write(char *file_name, hid_t my_fapl) { char filename[1024]; hid_t file = -1; hid_t dataset=-1; /* dataset ID */ hid_t sid=-1; /* dataspace ID */ hid_t dcpl=-1; /* dataset creation property list ID */ hsize_t dims[1]={DIM}; /* dataspace dimension - 10*/ hsize_t chunk_dims[1]={FILTER_CHUNK_DIM}; /* chunk dimension - 2*/ int nfilters; /* number of filters in DCPL */ unsigned flags; /* flags for filter */ int points[DIM]; /* Data */ int rbuf[DIM]; /* Data */ herr_t ret; /* generic return value */ int i; TESTING("data writing when a mandatory filter fails"); /* Create file */ if((file = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) TEST_ERROR /* create the data space */ if((sid = H5Screate_simple(1, dims, NULL)) < 0) TEST_ERROR /* Create dcpl and register the filter */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR if(H5Pset_chunk(dcpl, 1, chunk_dims) < 0) TEST_ERROR if(H5Zregister (H5Z_FAIL_TEST) < 0) TEST_ERROR /* Check that the filter was registered */ if(TRUE != H5Zfilter_avail(H5Z_FILTER_FAIL_TEST)) FAIL_STACK_ERROR /* Enable the filter as mandatory */ if(H5Pset_filter(dcpl, H5Z_FILTER_FAIL_TEST, 0, (size_t)0, NULL) < 0) TEST_ERROR /* create a dataset */ if((dataset = H5Dcreate2(file, DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) TEST_ERROR /* Initialize the write buffer */ for(i = 0; i < DIM; i++) points[i] = i; /* Write data */ if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, sid, H5P_DEFAULT, points) < 0) TEST_ERROR /* clean up objects used for this test */ if(H5Pclose (dcpl) < 0) TEST_ERROR if(H5Sclose (sid) < 0) TEST_ERROR /* Dataset closing should fail */ H5E_BEGIN_TRY { ret = H5Dclose (dataset); } H5E_END_TRY; if(ret >= 0) { H5_FAILED(); puts(" Dataset is supposed to fail because the chunk can't be flushed to file."); TEST_ERROR }