sz_params* H5Z_SZ_Init_Default()
{
herr_t ret = H5Zregister(H5Z_SZ);
sz_params* conf_params = (sz_params *)malloc(sizeof(sz_params));
confparams_cpr->quantization_intervals = 0;
confparams_cpr->max_quant_intervals = 65536;
dataEndianType = LITTLE_ENDIAN_DATA;
confparams_cpr->sol_ID = SZ;
//confparams_cpr->layers = 1;
confparams_cpr->sampleDistance = 100;
confparams_cpr->predThreshold = 0.99;
//confparams_cpr->offset = 0;
confparams_cpr->szMode = SZ_BEST_COMPRESSION;
confparams_cpr->gzipMode = 1; //best speed
confparams_cpr->errorBoundMode = REL; //details about errorBoundMode can be found in sz.config
confparams_cpr->absErrBound = 1E-4;
confparams_cpr->relBoundRatio = 1E-3;
confparams_cpr->pw_relBoundRatio = 1E-4;
confparams_cpr->segment_size = 32;
confparams_cpr->pwr_type = SZ_PWR_AVG_TYPE;
int status = SZ_Init_Params(conf_params);
if(status == SZ_NSCS || ret < 0)
return NULL;
else
return confparams_cpr;
}
int H5Z_SZ_Init_Params(sz_params *params)
{
herr_t ret = H5Zregister(H5Z_SZ);
int status = SZ_Init_Params(params);
if(status == SZ_NSCS || ret < 0)
return SZ_NSCS;
else
return SZ_SCES;
}
/*-------------------------------------------------------------------------
* Function: create_file_with_bogus_filter
*
* Purpose: Create a file and a dataset with a bogus filter enabled
*
* Return: Success: 0
*
* Failure: -1
*
* Programmer: Raymond Lu
* 2 June 2011
*
*-------------------------------------------------------------------------
*/
static herr_t
create_file_with_bogus_filter(void)
{
hid_t fid = -1; /* file ID */
hid_t dsid = -1; /* dataset ID */
hid_t sid = -1; /* dataspace ID */
hid_t dcpl = -1; /* dataset creation property list ID */
hsize_t dims[1] = {20}; /* dataspace dimensions */
hsize_t chunk_dims[1] = {10}; /* chunk dimensions */
int buf[20];
int rank = 1;
int i;
for(i = 0; i < 20; i++)
buf[i] = 1;
/* create a file using default properties */
if((fid = H5Fcreate(TESTFILE2, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error;
/* create a data space */
if((sid = H5Screate_simple(rank, dims, NULL)) < 0) goto error;
/* create dcpl */
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error;
/* create chunking */
if(H5Pset_chunk(dcpl, rank, chunk_dims) < 0) goto error;
/* register bogus filter */
if(H5Zregister (H5Z_BOGUS) < 0) goto error;
if(H5Pset_filter(dcpl, H5Z_FILTER_BOGUS, 0, (size_t)0, NULL) < 0) goto error;
/* create a dataset */
if((dsid = H5Dcreate2(fid, DSETNAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error;
if(H5Dwrite(dsid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) goto error;
/* close */
if(H5Pclose(dcpl) < 0) goto error;
if(H5Dclose(dsid) < 0) goto error;
if(H5Sclose(sid) < 0) goto error;
if(H5Fclose(fid) < 0) goto error;
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(dsid);
H5Sclose(sid);
H5Fclose(fid);
} H5E_END_TRY;
return -1;
}
int register_lzo(char **version, char **date) {
#ifdef HAVE_LZO_LIB
/* The conditional below is somewhat messy, but it is necessary because
the THG team has decided to fix an API inconsistency in the definition
of the H5Z_class_t structure in version 1.8.3 */
#if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 7) || \
(H5_USE_16_API && (H5_VERS_MAJOR > 1 || \
(H5_VERS_MAJOR == 1 && (H5_VERS_MINOR > 8 || \
(H5_VERS_MINOR == 8 && H5_VERS_RELEASE >= 3)))))
/* 1.6.x */
H5Z_class_t filter_class = {
(H5Z_filter_t)(FILTER_LZO), /* filter_id */
"lzo", /* comment */
NULL, /* can_apply_func */
NULL, /* set_local_func */
(H5Z_func_t)(lzo_deflate) /* filter_func */
};
#else
/* 1.8.x where x < 3 */
H5Z_class_t filter_class = {
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)(FILTER_LZO), /* filter_id */
1, 1, /* Encoding and decoding enabled */
"lzo", /* comment */
NULL, /* can_apply_func */
NULL, /* set_local_func */
(H5Z_func_t)(lzo_deflate) /* filter_func */
};
#endif
/* Init the LZO library */
if (lzo_init()!=LZO_E_OK) {
fprintf(stderr, "Problems initializing LZO library\n");
*version = NULL;
*date = NULL;
return 0; /* lib is not available */
}
/* Register the lzo compressor */
H5Zregister(&filter_class);
*version = strdup(LZO_VERSION_STRING);
*date = strdup(LZO_VERSION_DATE);
return 1; /* lib is available */
#else
*version = NULL;
*date = NULL;
return 0; /* lib is not available */
#endif /* HAVE_LZO_LIB */
}
/*-------------------------------------------------------------------------
* Function: create_dataset
*
* Purpose: Creates a square dataset with square chunks, registers a
* stupid compress/uncompress pair for counting I/O, and
* initializes the dataset. The chunk size is in bytes, the
* dataset size is in terms of chunks.
*
* Return: void
*
* Programmer: Robb Matzke
* Thursday, May 14, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static void
create_dataset (void)
{
hid_t file, space, dcpl, dset;
hsize_t size[2];
signed char *buf;
/* The file */
file = H5Fcreate (FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_g);
/* The data space */
size[0] = size[1] = DS_SIZE * CH_SIZE;
space = H5Screate_simple (2, size, size);
/* The storage layout and compression */
dcpl = H5Pcreate (H5P_DATASET_CREATE);
size[0] = size[1] = CH_SIZE;
H5Pset_chunk (dcpl, 2, size);
#ifdef H5_WANT_H5_V1_4_COMPAT
H5Zregister (FILTER_COUNTER, "counter", counter);
#else /* H5_WANT_H5_V1_4_COMPAT */
H5Zregister (H5Z_COUNTER);
#endif /* H5_WANT_H5_V1_4_COMPAT */
H5Pset_filter (dcpl, FILTER_COUNTER, 0, 0, NULL);
/* The dataset */
dset = H5Dcreate (file, "dset", H5T_NATIVE_SCHAR, space, dcpl);
assert (dset>=0);
/* The data */
buf = calloc (1, SQUARE (DS_SIZE*CH_SIZE));
H5Dwrite (dset, H5T_NATIVE_SCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf);
free (buf);
/* Close */
H5Dclose (dset);
H5Sclose (space);
H5Pclose (dcpl);
H5Fclose (file);
}
/*-------------------------------------------------------------------------
* Function: test_filter_write_failure
*
* Purpose: Tests the library's behavior when a mandate filter returns
* failure. There're only 5 chunks with each of them having
* 2 integers. The filter will fail in the last chunk. The
* dataset should release all resources even though the last
* chunk can't be flushed to file. The file should close
* successfully.
*
* Return:
* Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 25 August 2010
*
* Modifications:
* Raymond Lu
* 5 Oct 2010
* Test when the chunk cache is enable and disabled to make
* sure the library behaves properly.
*-------------------------------------------------------------------------
*/
static herr_t
test_filter_write(char *file_name, hid_t my_fapl, hbool_t cache_enabled)
{
hid_t file = -1;
hid_t dataset=-1; /* dataset ID */
hid_t sid=-1; /* dataspace ID */
hid_t dcpl=-1; /* dataset creation property list ID */
hsize_t dims[1]={DIM}; /* dataspace dimension - 10*/
hsize_t chunk_dims[1]={FILTER_CHUNK_DIM}; /* chunk dimension - 2*/
int points[DIM]; /* Data */
herr_t ret; /* generic return value */
int i;
if(cache_enabled) {
TESTING("data writing when a mandatory filter fails and chunk cache is enabled");
} else {
TESTING("data writing when a mandatory filter fails and chunk cache is disabled");
}
/* Create file */
if((file = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) TEST_ERROR
/* create the data space */
if((sid = H5Screate_simple(1, dims, NULL)) < 0) TEST_ERROR
/* Create dcpl and register the filter */
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR
if(H5Pset_chunk(dcpl, 1, chunk_dims) < 0) TEST_ERROR
if(H5Zregister (H5Z_FAIL_TEST) < 0) TEST_ERROR
/* Check that the filter was registered */
if(TRUE != H5Zfilter_avail(H5Z_FILTER_FAIL_TEST)) FAIL_STACK_ERROR
/* Enable the filter as mandatory */
if(H5Pset_filter(dcpl, H5Z_FILTER_FAIL_TEST, 0, (size_t)0, NULL) < 0)
TEST_ERROR
/* create a dataset */
if((dataset = H5Dcreate2(file, DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) TEST_ERROR
/* Initialize the write buffer */
for(i = 0; i < DIM; i++)
points[i] = i;
/* Write data. If the chunk cache is enabled, H5Dwrite should succeed. If it is
* diabled, H5Dwrite should fail. */
if(cache_enabled) {
if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, sid, H5P_DEFAULT, points) < 0)
TEST_ERROR
} else {
loaded_hdf5_plugin() {
int reg_rval = 0;
if (!H5Zfilter_avail(H5Z_FILTER_SQY))
reg_rval = H5Zregister(H5Z_SQY);
if (reg_rval < 0)
std::cerr << __FILE__ << ":"<< __LINE__
<< "\t unable to register sqy as hdf5 filter!!\n";
else
std::cout << "Done registering sqy as hdf5 filter.\n";
}
int H5Z_SZ_Init(char* cfgFile)
{
herr_t ret;
//printf("start in H5Z_SZ_Init, load_conffile_flag = %d\n", load_conffile_flag);
if(load_conffile_flag==0)
{
load_conffile_flag = 1;
int status = SZ_Init(cfgFile);
//printf("cfgFile=%s\n", cfgFile);
//printf("szMode=%d, errorBoundMode=%d, relBoundRatio=%f\n", szMode, errorBoundMode, relBoundRatio);
if(status == SZ_NSCS)
return SZ_NSCS;
else
return SZ_SCES;
}
ret = H5Zregister(H5Z_SZ);
if(ret < 0)
return SZ_NSCS;
else
return SZ_SCES;
}
int register_bzip2(char **version, char **date)
{
#ifdef HAVE_BZ2_LIB
char *libver, *versionp, *datep, *sep;
H5Z_class_t filter_class = {
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)(FILTER_BZIP2), /* filter_id */
1, 1, /* Encoding and decoding enabled */
"bzip2", /* comment */
NULL, /* can_apply_func */
NULL, /* set_local_func */
(H5Z_func_t)(bzip2_deflate) /* filter_func */
};
/* Register the filter class for the bzip2 compressor. */
H5Zregister(&filter_class);
/* Get the library major version from the version string. */
libver = strdup(BZ2_bzlibVersion());
sep = strchr(libver, ',');
assert(sep != NULL);
assert(*(sep + 1) == ' ');
*sep = '\0';
versionp = libver;
datep = sep + 2; /* after the comma and a space */
*version = strdup(versionp);
*date = strdup(datep);
free(libver);
return 1; /* library is available */
#else
return 0; /* library is not available */
#endif /* defined HAVE_BZ2_LIB */
}
int register_lzo(char **version, char **date) {
#ifdef HAVE_LZO_LIB
H5Z_class_t filter_class = {
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)(FILTER_LZO), /* filter_id */
1, 1, /* Encoding and decoding enabled */
"lzo", /* comment */
NULL, /* can_apply_func */
NULL, /* set_local_func */
(H5Z_func_t)(lzo_deflate) /* filter_func */
};
/* Init the LZO library */
if (lzo_init()!=LZO_E_OK) {
fprintf(stderr, "Problems initializing LZO library\n");
*version = NULL;
*date = NULL;
return 0; /* lib is not available */
}
/* Register the lzo compressor */
H5Zregister(&filter_class);
*version = strdup(LZO_VERSION_STRING);
*date = strdup(LZO_VERSION_DATE);
return 1; /* lib is available */
#else
*version = NULL;
*date = NULL;
return 0; /* lib is not available */
#endif /* HAVE_LZO_LIB */
}
/*-------------------------------------------------------------------------
* Function: test_skip_compress_write2
*
* Purpose: Test skipping compression filter when there are three filters
* for the dataset
*
* Return: Success: 0
*
* Failure: 1
*
* Programmer: Raymond Lu
* 30 November 2012
*
*-------------------------------------------------------------------------
*/
static int
test_skip_compress_write2(hid_t file)
{
hid_t dataspace = -1, dataset = -1;
hid_t mem_space = -1;
hid_t cparms = -1, dxpl = -1;
hsize_t dims[2] = {NX, NY};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY};
herr_t status;
int i, j, n;
unsigned filter_mask = 0;
int origin_direct_buf[CHUNK_NX][CHUNK_NY];
int direct_buf[CHUNK_NX][CHUNK_NY];
int check_chunk[CHUNK_NX][CHUNK_NY];
hsize_t offset[2] = {0, 0};
size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
int aggression = 9; /* Compression aggression setting */
hsize_t start[2]; /* Start of hyperslab */
hsize_t stride[2]; /* Stride of hyperslab */
hsize_t count[2]; /* Block count */
hsize_t block[2]; /* Block sizes */
TESTING("skipping compression filters but keep two other filters");
/*
* Create the data space with unlimited dimensions.
*/
if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0)
goto error;
if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0)
goto error;
/*
* Modify dataset creation properties, i.e. enable chunking and compression.
* The order of filters is bogus 1 + deflate + bogus 2.
*/
if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0)
goto error;
/* Register and enable first bogus filter */
if(H5Zregister (H5Z_BOGUS1) < 0)
goto error;
if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS1, 0, (size_t)0, NULL) < 0)
goto error;
/* Enable compression filter */
if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0)
goto error;
/* Register and enable second bogus filter */
if(H5Zregister (H5Z_BOGUS2) < 0)
goto error;
if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS2, 0, (size_t)0, NULL) < 0)
goto error;
/*
* Create a new dataset within the file using cparms
* creation properties.
*/
if((dataset = H5Dcreate2(file, DATASETNAME3, H5T_NATIVE_INT, dataspace, H5P_DEFAULT,
cparms, H5P_DEFAULT)) < 0)
goto error;
if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0)
goto error;
/* Initialize data for one chunk. Apply operations of two bogus filters to the chunk */
for(i = n = 0; i < CHUNK_NX; i++)
for(j = 0; j < CHUNK_NY; j++) {
origin_direct_buf[i][j] = n++;
direct_buf[i][j] = (origin_direct_buf[i][j] + ADD_ON) * FACTOR;
}
/* write the uncompressed chunk data repeatedly to dataset, using the direct writing function.
* Indicate skipping the compression filter but keep the other two bogus filters */
offset[0] = CHUNK_NX;
offset[1] = CHUNK_NY;
/* compression filter is the middle one to be skipped */
filter_mask = 0x00000002;
if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0)
goto error;
if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0)
goto error;
if(H5Dclose(dataset) < 0)
goto error;
if((dataset = H5Dopen2(file, DATASETNAME3, H5P_DEFAULT)) < 0)
goto error;
/*
* Select hyperslab for one chunk in the file
*/
start[0] = CHUNK_NX; start[1] = CHUNK_NY;
stride[0] = 1; stride[1] = 1;
count[0] = 1; count[1] = 1;
block[0] = CHUNK_NX; block[1] = CHUNK_NY;
if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0)
goto error;
/* Read the chunk back */
if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0)
goto error;
/* Check that the values read are the same as the values written */
for(i = 0; i < CHUNK_NX; i++) {
for(j = 0; j < CHUNK_NY; j++) {
if(origin_direct_buf[i][j] != check_chunk[i][j]) {
printf(" 1. Read different values than written.");
printf(" At index %d,%d\n", i, j);
printf(" origin_direct_buf=%d, check_chunk=%d\n", origin_direct_buf[i][j], check_chunk[i][j]);
goto error;
}
}
}
/*
* Close/release resources.
*/
H5Dclose(dataset);
H5Sclose(mem_space);
H5Sclose(dataspace);
H5Pclose(cparms);
H5Pclose(dxpl);
PASSED();
return 0;
error:
H5E_BEGIN_TRY {
H5Dclose(dataset);
H5Sclose(mem_space);
H5Sclose(dataspace);
H5Pclose(cparms);
H5Pclose(dxpl);
} H5E_END_TRY;
return 1;
}
void cbf_register_filters() {
H5Zregister(&H5Z_LZ4);
H5Zregister(&bshuf_H5Filter);
}
/*-------------------------------------------------------------------------
* Function: test_filter_write_failure
*
* Purpose: Tests the library's behavior when a mandate filter returns
* failure. There're only 5 chunks with each of them having
* 2 integers. The filter will fail in the last chunk. The
* dataset should release all resources even though the last
* chunk can't be flushed to file. The file should close
* successfully.
*
* Return:
* Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 25 August 2010
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
test_filter_write(char *file_name, hid_t my_fapl)
{
char filename[1024];
hid_t file = -1;
hid_t dataset=-1; /* dataset ID */
hid_t sid=-1; /* dataspace ID */
hid_t dcpl=-1; /* dataset creation property list ID */
hsize_t dims[1]={DIM}; /* dataspace dimension - 10*/
hsize_t chunk_dims[1]={FILTER_CHUNK_DIM}; /* chunk dimension - 2*/
int nfilters; /* number of filters in DCPL */
unsigned flags; /* flags for filter */
int points[DIM]; /* Data */
int rbuf[DIM]; /* Data */
herr_t ret; /* generic return value */
int i;
TESTING("data writing when a mandatory filter fails");
/* Create file */
if((file = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) TEST_ERROR
/* create the data space */
if((sid = H5Screate_simple(1, dims, NULL)) < 0) TEST_ERROR
/* Create dcpl and register the filter */
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR
if(H5Pset_chunk(dcpl, 1, chunk_dims) < 0) TEST_ERROR
if(H5Zregister (H5Z_FAIL_TEST) < 0) TEST_ERROR
/* Check that the filter was registered */
if(TRUE != H5Zfilter_avail(H5Z_FILTER_FAIL_TEST)) FAIL_STACK_ERROR
/* Enable the filter as mandatory */
if(H5Pset_filter(dcpl, H5Z_FILTER_FAIL_TEST, 0, (size_t)0, NULL) < 0)
TEST_ERROR
/* create a dataset */
if((dataset = H5Dcreate2(file, DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) TEST_ERROR
/* Initialize the write buffer */
for(i = 0; i < DIM; i++)
points[i] = i;
/* Write data */
if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, sid, H5P_DEFAULT, points) < 0) TEST_ERROR
/* clean up objects used for this test */
if(H5Pclose (dcpl) < 0) TEST_ERROR
if(H5Sclose (sid) < 0) TEST_ERROR
/* Dataset closing should fail */
H5E_BEGIN_TRY {
ret = H5Dclose (dataset);
} H5E_END_TRY;
if(ret >= 0) {
H5_FAILED();
puts(" Dataset is supposed to fail because the chunk can't be flushed to file.");
TEST_ERROR
}
