//--------------------------------------------------------------------------
// Function: DSetCreatPropList::setSzip
///\brief Sets up for the use of the SZIP compression filter.
///\param options_mask - IN: A bit-mask conveying the desired SZIP
/// options. Valid values are H5_SZIP_EC_OPTION_MASK
/// and H5_SZIP_NN_OPTION_MASK.
///\param pixels_per_block - IN: Number of pixels or data elements in
/// each data block.
///\exception H5::PropListIException
///\par Description
/// The associate C function sets an SZIP compression filter,
/// H5Z_FILTER_SZIP, for a dataset. For more information about
/// SZIP and usage, please refer to the C layer Reference
/// Manual at:
/// http://hdfgroup.org/HDF5/doc/RM_H5P.html#Property-SetSzip
// Programmer Binh-Minh Ribler - Jan, 2007
//--------------------------------------------------------------------------
void DSetCreatPropList::setSzip(unsigned int options_mask, unsigned int pixels_per_block) const
{
herr_t ret_value = H5Pset_szip(id, options_mask, pixels_per_block);
if( ret_value < 0 )
{
throw PropListIException("DSetCreatPropList::setSzip",
"H5Pset_szip failed");
}
}
/**
Create a new 1D dataset. Make it extensible.
*/
hid_t HDF5DataWriter::create_dataset(hid_t parent_id, string name)
{
herr_t status;
hsize_t dims[1] = {0};
hsize_t maxdims[] = {H5S_UNLIMITED};
hsize_t chunk_dims[] = {chunkSize_};
hid_t chunk_params = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk(chunk_params, 1, chunk_dims);
assert( status >= 0 );
if (compressor_ == "zlib"){
status = H5Pset_deflate(chunk_params, compression_);
} else if (compressor_ == "szip"){
// this needs more study
unsigned sz_opt_mask = H5_SZIP_NN_OPTION_MASK;
status = H5Pset_szip(chunk_params, sz_opt_mask, HDF5WriterBase::CHUNK_SIZE);
}
hid_t dataspace = H5Screate_simple(1, dims, maxdims);
hid_t dataset_id = H5Dcreate2(parent_id, name.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, chunk_params, H5P_DEFAULT);
return dataset_id;
}
int apply_filters(const char* name, /* object name from traverse list */
int rank, /* rank of dataset */
hsize_t *dims, /* dimensions of dataset */
size_t msize, /* size of type */
hid_t dcpl_id, /* dataset creation property list */
pack_opt_t *options, /* repack options */
int *has_filter) /* (OUT) object NAME has a filter */
{
int nfilters; /* number of filters in DCPL */
hsize_t chsize[64]; /* chunk size in elements */
H5D_layout_t layout;
int i;
pack_info_t obj;
*has_filter = 0;
if (rank==0) /* scalar dataset, do not apply */
return 0;
/*-------------------------------------------------------------------------
* initialize the assigment object
*-------------------------------------------------------------------------
*/
init_packobject(&obj);
/*-------------------------------------------------------------------------
* find options
*-------------------------------------------------------------------------
*/
if (aux_assign_obj(name,options,&obj)==0)
return 0;
/* get information about input filters */
if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
return -1;
/*-------------------------------------------------------------------------
* check if we have filters in the pipeline
* we want to replace them with the input filters
* only remove if we are inserting new ones
*-------------------------------------------------------------------------
*/
if (nfilters && obj.nfilters )
{
*has_filter = 1;
if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0)
return -1;
}
/*-------------------------------------------------------------------------
* check if there is an existent chunk
* read it only if there is not a requested layout
*-------------------------------------------------------------------------
*/
if (obj.layout == -1 )
{
if ((layout = H5Pget_layout(dcpl_id))<0)
return -1;
if (layout == H5D_CHUNKED)
{
if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0)
return -1;
obj.layout = H5D_CHUNKED;
obj.chunk.rank = rank;
for ( i = 0; i < rank; i++)
obj.chunk.chunk_lengths[i] = chsize[i];
}
}
/*-------------------------------------------------------------------------
* the type of filter and additional parameter
* type can be one of the filters
* H5Z_FILTER_NONE 0 , uncompress if compressed
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
* H5Z_FILTER_SZIP 4 , szip compression
* H5Z_FILTER_NBIT 5 , nbit compression
* H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression
*-------------------------------------------------------------------------
*/
if (obj.nfilters)
{
/*-------------------------------------------------------------------------
* filters require CHUNK layout; if we do not have one define a default
*-------------------------------------------------------------------------
*/
if (obj.layout==-1)
{
/* stripmine info */
hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */
hsize_t sm_nbytes; /*bytes per stripmine */
obj.chunk.rank = rank;
/*
* determine the strip mine size. The strip mine is
* a hyperslab whose size is manageable.
*/
sm_nbytes = msize;
for ( i = rank; i > 0; --i)
{
hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
size = 1;
sm_size[i - 1] = MIN(dims[i - 1], size);
sm_nbytes *= sm_size[i - 1];
assert(sm_nbytes > 0);
}
for ( i = 0; i < rank; i++)
{
obj.chunk.chunk_lengths[i] = sm_size[i];
}
}
for ( i=0; i<obj.nfilters; i++)
{
switch (obj.filter[i].filtn)
{
default:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_DEFLATE:
{
unsigned aggression; /* the deflate level */
aggression = obj.filter[i].cd_values[0];
/* set up for deflated data */
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if(H5Pset_deflate(dcpl_id,aggression)<0)
return -1;
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SZIP 4 , szip compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SZIP:
{
unsigned options_mask;
unsigned pixels_per_block;
options_mask = obj.filter[i].cd_values[0];
pixels_per_block = obj.filter[i].cd_values[1];
/* set up for szip data */
if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0)
return -1;
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SHUFFLE:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_shuffle(dcpl_id)<0)
return -1;
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_FLETCHER32:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_fletcher32(dcpl_id)<0)
return -1;
break;
/*----------- -------------------------------------------------------------
* H5Z_FILTER_NBIT , NBIT compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_NBIT:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_nbit(dcpl_id)<0)
return -1;
break;
/*----------- -------------------------------------------------------------
* H5Z_FILTER_SCALEOFFSET , scale+offset compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SCALEOFFSET:
{
H5Z_SO_scale_type_t scale_type;
int scale_factor;
scale_type = (H5Z_SO_scale_type_t)obj.filter[i].cd_values[0];
scale_factor = obj.filter[i].cd_values[1];
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0)
return -1;
}
break;
} /* switch */
}/*i*/
}
/*obj.nfilters*/
/*-------------------------------------------------------------------------
* layout
*-------------------------------------------------------------------------
*/
if (obj.layout>=0)
{
/* a layout was defined */
if (H5Pset_layout(dcpl_id, obj.layout)<0)
return -1;
if (H5D_CHUNKED == obj.layout)
{
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
}
else if (H5D_COMPACT == obj.layout)
{
if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY)<0)
return -1;
}
/* remove filters for the H5D_CONTIGUOUS case */
else if (H5D_CONTIGUOUS == obj.layout)
{
if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0)
return -1;
}
}
return 0;
}
int
main(int argc, char **argv)
{
printf("\n*** Testing HDF5/NetCDF-4 interoperability...\n");
printf("*** testing HDF5 compatibility...");
{
#define GRPA_NAME "grpa"
#define VAR_NAME "vara"
#define NDIMS 2
int nrowCur = 7; /* current size */
int ncolCur = 3;
int nrowMax = nrowCur + 0; /* maximum size */
int ncolMax = ncolCur + 0;
hid_t xdimId;
hid_t ydimId;
hsize_t xscaleDims[1];
hsize_t yscaleDims[1];
hid_t xdimSpaceId, spaceId;
hid_t fileId;
hid_t fapl;
hsize_t curDims[2];
hsize_t maxDims[2];
hid_t dataTypeId, dsPropertyId, grpaId, grpaPropId, dsId;
hid_t ydimSpaceId;
const char * dimNameBase
= "This is a netCDF dimension but not a netCDF variable.";
char dimNameBuf[1000];
char *varaName = "/grpa/vara";
short amat[nrowCur][ncolCur];
int ii, jj;
xscaleDims[0] = nrowCur;
yscaleDims[0] = ncolCur;
if ((xdimSpaceId = H5Screate_simple(1, xscaleDims, NULL)) < 0) ERR;
/* With the SEMI close degree, the HDF5 file close will fail if
* anything is left open. */
if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR;
if (H5Pset_fclose_degree(fapl, H5F_CLOSE_SEMI)) ERR;
/* Create file */
if((fileId = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC,
H5Pcreate(H5P_FILE_CREATE), fapl)) < 0) ERR;
if (H5Pclose(fapl) < 0) ERR;
/* Create data space */
curDims[0] = nrowCur;
curDims[1] = ncolCur;
maxDims[0] = nrowMax;
maxDims[1] = ncolMax;
if ((spaceId = H5Screate_simple(2, curDims, maxDims)) < 0) ERR;
if ((dataTypeId = H5Tcopy(H5T_NATIVE_SHORT)) < 0) ERR;
if ((dsPropertyId = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
if ((grpaPropId = H5Pcreate(H5P_GROUP_CREATE)) < 0) ERR;
if ((grpaId = H5Gcreate2(fileId, GRPA_NAME, H5P_DEFAULT,
grpaPropId, H5P_DEFAULT)) < 0) ERR;
if (H5Pclose(grpaPropId) < 0) ERR;
/* Create vara dataset */
if ((dsId = H5Dcreate2(fileId, varaName, dataTypeId, spaceId,
H5P_DEFAULT, dsPropertyId,
H5P_DEFAULT)) < 0) ERR;
if (H5Pclose(dsPropertyId) < 0) ERR;
if (H5Tclose(dataTypeId) < 0) ERR;
if ((ydimSpaceId = H5Screate_simple(1, yscaleDims, NULL)) < 0) ERR;
/* Create xdim dimension dataset */
if ((xdimId = H5Dcreate2(fileId, "/xdim", H5T_IEEE_F32BE,
xdimSpaceId, H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT)) < 0) ERR;
if (H5Sclose(xdimSpaceId) < 0) ERR;
/* Create ydim dimension dataset */
if ((ydimId = H5Dcreate2(fileId, "/ydim", H5T_IEEE_F32BE,
ydimSpaceId, H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT)) < 0) ERR;
if (H5Sclose(ydimSpaceId) < 0) ERR;
/* Create xdim scale */
sprintf(dimNameBuf, "%s%10d", dimNameBase, nrowCur);
if (H5DSset_scale(xdimId, dimNameBuf) < 0) ERR;
/* Create ydim scale */
sprintf(dimNameBuf, "%s%10d", dimNameBase, ncolCur);
if (H5DSset_scale(ydimId, dimNameBuf) < 0) ERR;
/* Attach dimension scales to the dataset */
if (H5DSattach_scale(dsId, xdimId, 0) < 0) ERR;
if (H5DSattach_scale(dsId, ydimId, 1) < 0) ERR;
/* Close stuff. */
if (H5Dclose(xdimId) < 0) ERR;
if (H5Dclose(ydimId) < 0) ERR;
if (H5Dclose(dsId) < 0) ERR;
if (H5Gclose(grpaId) < 0) ERR;
if (H5Sclose(spaceId) < 0) ERR;
if (H5Fclose(fileId) < 0) ERR;
/* Create some data */
for (ii = 0; ii < nrowCur; ii++)
for (jj = 0; jj < ncolCur; jj++)
amat[ii][jj] = 100 * ii + jj;
/* Re-open file */
if ((fileId = H5Fopen(FILE_NAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) ERR;
if ((grpaId = H5Gopen2(fileId, GRPA_NAME, H5P_DEFAULT)) < 0) ERR;
if ((dsId = H5Dopen2(grpaId, varaName, H5P_DEFAULT)) < 0) ERR;
/* Write dataset */
if (H5Dwrite(dsId, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
amat) < 0) ERR;
/* Write dimension values for both xdim, ydim */
{
short xydimMat[ nrowCur >= ncolCur ? nrowCur : ncolCur];
for (ii = 0; ii < nrowCur; ii++)
xydimMat[ii] = 0; /*#### 100 * ii; */
/* Write xdim */
if ((xdimId = H5Dopen2(fileId, "/xdim", H5P_DEFAULT)) < 0) ERR;
if (H5Dwrite(xdimId, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, xydimMat) < 0) ERR;
if (H5Dclose(xdimId) < 0) ERR;
/* Write ydim */
if ((ydimId = H5Dopen2(fileId, "/ydim", H5P_DEFAULT)) < 0) ERR;
if (H5Dwrite(ydimId, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, xydimMat) < 0) ERR;
if (H5Dclose(ydimId) < 0) ERR;
}
if (H5Dclose(dsId) < 0) ERR;
if (H5Gclose(grpaId) < 0) ERR;
if (H5Fclose(fileId) < 0) ERR;
{
int ncid, grpid, nvars, ngatts, ndims, unlimdimid, ngrps;
char name_in[NC_MAX_NAME + 1];
nc_type xtype_in;
int ndims_in, natts_in, dimid_in[NDIMS];
/* nc_set_log_level(5);*/
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (ndims != 2 || nvars != 0 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_grps(ncid, &ngrps, &grpid)) ERR;
if (ngrps != 1) ERR;
if (nc_inq(grpid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (ndims != 0 || nvars != 1 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_var(grpid, 0, name_in, &xtype_in, &ndims_in, dimid_in,
&natts_in)) ERR;
if (strcmp(name_in, VAR_NAME) || xtype_in != NC_SHORT || ndims_in != NDIMS ||
dimid_in[0] != 0 || dimid_in[1] != 1 || natts_in != 0) ERR;
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
#ifdef USE_SZIP
printf("*** testing HDF5 compatibility with szip...");
{
#define DEFLATE_LEVEL 9
#define MAX_NAME 100
#define NUM_CD_ELEM 10
/* HDF5 defines this... */
#define DEFLATE_NAME "deflate"
#define DIM1_LEN 3000
#define GRP_NAME "George_Washington"
#define BATTLE_RECORD "Battle_Record"
hid_t fileid, grpid, spaceid, datasetid;
int data_out[DIM1_LEN], data_in[DIM1_LEN];
hsize_t dims[1] = {DIM1_LEN};
hid_t propid;
char name_in[MAX_NAME + 1];
int ncid, ndims_in, nvars_in, ngatts_in, unlimdimid_in, ngrps_in;
int nc_grpid;
int dimid_in[1], natts_in;
nc_type xtype_in;
int i;
for (i = 0; i < DIM1_LEN; i++)
data_out[i] = i;
/* Open file and create group. */
if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT,
H5P_DEFAULT)) < 0) ERR;
if ((grpid = H5Gcreate(fileid, GRP_NAME, 0)) < 0) ERR;
/* Write an array of bools, with szip compression. */
if ((propid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
if (H5Pset_layout(propid, H5D_CHUNKED)) ERR;
if (H5Pset_chunk(propid, 1, dims)) ERR;
if (H5Pset_szip(propid, H5_SZIP_EC_OPTION_MASK, 32)) ERR;
if ((spaceid = H5Screate_simple(1, dims, dims)) < 0) ERR;
if ((datasetid = H5Dcreate(grpid, BATTLE_RECORD, H5T_NATIVE_INT,
spaceid, propid)) < 0) ERR;
if (H5Dwrite(datasetid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
data_out) < 0) ERR;
if (H5Dclose(datasetid) < 0 ||
H5Pclose(propid) < 0 ||
H5Sclose(spaceid) < 0 ||
H5Gclose(grpid) < 0 ||
H5Fclose(fileid) < 0)
ERR;
/* Open the file with netCDF and check it. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in)) ERR;
if (ndims_in != 0 || nvars_in != 0 || ngatts_in != 0 || unlimdimid_in != -1) ERR;
if (nc_inq_grps(ncid, &ngrps_in, &nc_grpid)) ERR;
if (ngrps_in != 1) ERR;
if (nc_inq(nc_grpid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in)) ERR;
if (ndims_in != 1 || nvars_in != 1 || ngatts_in != 0 || unlimdimid_in != -1) ERR;
/* Check the variable. */
if (nc_inq_var(nc_grpid, 0, name_in, &xtype_in, &ndims_in, dimid_in,
&natts_in)) ERR;
if (strcmp(name_in, BATTLE_RECORD) || xtype_in != NC_INT || ndims_in != 1 ||
dimid_in[0] != 0 || natts_in != 0) ERR;
/* Check the data. */
if (nc_get_var(nc_grpid, 0, data_in)) ERR;
for (i = 0; i < DIM1_LEN; i++)
if (data_in[i] != data_out[i]) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#endif /* USE_SZIP */
FINAL_RESULTS;
}
/*-------------------------------------------------------------------------
* Function: create_szip_dsets_float
*
* Purpose: Create a dataset of FLOAT datatype with szip filter
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 29 March 2011
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
create_szip_dsets_float(hid_t fid, hid_t fsid, hid_t msid)
{
hid_t dataset; /* dataset handles */
hid_t dcpl;
float data[NX][NY]; /* data to write */
float fillvalue = -2.2f;
hsize_t chunk[RANK] = {CHUNK0, CHUNK1};
int i, j;
/*
* Data and output buffer initialization.
*/
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++)
data[j][i] = ((float)(i + j + 1))/3;
}
/*
* Create the dataset creation property list, add the Scale-Offset
* filter, set the chunk size, and set the fill value.
*/
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_szip(dcpl, H5_SZIP_NN_OPTION_MASK, 4) < 0)
TEST_ERROR
if(H5Pset_chunk(dcpl, RANK, chunk) < 0)
TEST_ERROR
if(H5Pset_fill_value(dcpl, H5T_NATIVE_FLOAT, &fillvalue) < 0)
TEST_ERROR
/*
* Create a new dataset within the file using defined dataspace, little
* endian datatype and default dataset creation properties.
*/
if((dataset = H5Dcreate2(fid, DATASETNAME18, H5T_IEEE_F32LE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
/*
* Write the data to the dataset using default transfer properties.
*/
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
/* Close dataset */
if(H5Dclose(dataset) < 0)
TEST_ERROR
/* Now create a dataset with a big-endian type */
if((dataset = H5Dcreate2(fid, DATASETNAME19, H5T_IEEE_F32BE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
if(H5Dclose(dataset) < 0)
TEST_ERROR
/*
* Close/release resources.
*/
if(H5Pclose(dcpl) < 0)
TEST_ERROR
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(dataset);
} H5E_END_TRY;
return -1;
}
