/*-------------------------------------------------------------------------
* Function: options_table_init
*
* Purpose: init options table
*
* Return: 0, ok, -1, fail
*
*-------------------------------------------------------------------------
*/
int options_table_init( pack_opttbl_t **tbl )
{
unsigned int i;
pack_opttbl_t *table;
if(NULL == (table = (pack_opttbl_t *)malloc(sizeof(pack_opttbl_t))))
{
error_msg("not enough memory for options table\n");
return -1;
}
table->size = 30;
table->nelems = 0;
if(NULL == (table->objs = (pack_info_t*)malloc(table->size * sizeof(pack_info_t))))
{
error_msg("not enough memory for options table\n");
free(table);
return -1;
}
for(i = 0; i < table->size; i++)
init_packobject(&table->objs[i]);
*tbl = table;
return 0;
}
/*-------------------------------------------------------------------------
* Function: h5repack_addlayout
*
* Purpose: add a layout option
*
* Return: 0, ok, -1, fail
*-------------------------------------------------------------------------
*/
int
h5repack_addlayout(const char *str, pack_opt_t *options)
{
obj_list_t *obj_list = NULL; /*one object list for the -t and -c option entry */
unsigned n_objs; /*number of objects in the current -t or -c option entry */
pack_info_t pack; /*info about layout to extract from parse */
int j;
int ret_value = -1;
init_packobject(&pack);
if (options->all_layout == 1) {
error_msg( "invalid layout input: 'all' option is present with other objects <%s>\n", str);
return ret_value;
}
/* parse the layout option */
obj_list = parse_layout(str, &n_objs, &pack, options);
if (obj_list) {
/* set layout option */
options->layout_g = pack.layout;
/* no individual dataset specified */
if (options->all_layout == 1) {
if (pack.layout == H5D_CHUNKED) {
/* -2 means the NONE option, remove chunking
and set the global layout to contiguous */
if (pack.chunk.rank == -2)
options->layout_g = H5D_CONTIGUOUS;
/* otherwise set the global chunking type */
else {
options->chunk_g.rank = pack.chunk.rank;
for (j = 0; j < pack.chunk.rank; j++)
options->chunk_g.chunk_lengths[j] = pack.chunk.chunk_lengths[j];
}
}
}
/* individual dataset specified */
if (options->all_layout == 0)
ret_value = options_add_layout(obj_list, n_objs, &pack, options->op_tbl);
HDfree(obj_list);
ret_value = 0;
}
return ret_value;
}
int h5repack_addlayout(const char* str,
pack_opt_t *options)
{
obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */
int n_objs; /*number of objects in the current -t or -c option entry */
pack_info_t pack; /*info about layout to extract from parse */
int j;
init_packobject(&pack);
if (options->all_layout==1){
error_msg("invalid layout input: 'all' option \
is present with other objects <%s>\n",str);
return -1;
}
static int aux_inctable(pack_opttbl_t *table, int n_objs )
{
unsigned int i;
table->size += n_objs;
table->objs = (pack_info_t*)realloc(table->objs, table->size * sizeof(pack_info_t));
if (table->objs==NULL) {
error_msg("not enough memory for options table\n");
return -1;
}
for (i = table->nelems; i < table->size; i++)
{
init_packobject(&table->objs[i]);
}
return 0;
}
/*-------------------------------------------------------------------------
* Function: aux_inctable
*
* Purpose: auxiliary function, increases the size of the collection by N_OBJS
*
* Return: 0, ok, -1, fail
*
*-------------------------------------------------------------------------
*/
static int
aux_inctable(pack_opttbl_t *table, unsigned n_objs)
{
unsigned u;
table->size += n_objs;
table->objs = (pack_info_t*) HDrealloc(table->objs, table->size * sizeof(pack_info_t));
if (table->objs == NULL) {
error_msg("not enough memory for options table\n");
return -1;
}
for (u = table->nelems; u < table->size; u++)
init_packobject(&table->objs[u]);
return 0;
}
int
h5repack_verify(const char *out_fname, pack_opt_t *options)
{
int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */
hid_t fidout = -1; /* file ID for output file*/
hid_t did = -1; /* dataset ID */
hid_t pid = -1; /* dataset creation property list ID */
hid_t sid = -1; /* space ID */
hid_t tid = -1; /* type ID */
unsigned int i;
trav_table_t *travt = NULL;
int ok = 1;
/* open the output file */
if((fidout = H5Fopen(out_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 )
return -1;
for(i = 0; i < options->op_tbl->nelems; i++)
{
char* name = options->op_tbl->objs[i].path;
pack_info_t *obj = &options->op_tbl->objs[i];
/*-------------------------------------------------------------------------
* open
*-------------------------------------------------------------------------
*/
if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed");
if((sid = H5Dget_space(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed");
if((pid = H5Dget_create_plist(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed");
if((tid = H5Dget_type(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed");
/*-------------------------------------------------------------------------
* filter check
*-------------------------------------------------------------------------
*/
if(verify_filters(pid, tid, obj->nfilters, obj->filter) <= 0)
ok = 0;
/*-------------------------------------------------------------------------
* layout check
*-------------------------------------------------------------------------
*/
if((obj->layout != -1) && (verify_layout(pid, obj) == 0))
ok = 0;
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if(H5Pclose(pid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
if (H5Sclose(sid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed");
if (H5Dclose(did) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
if (H5Tclose(tid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed");
}
/*-------------------------------------------------------------------------
* check for the "all" objects option
*-------------------------------------------------------------------------
*/
if(options->all_filter == 1 || options->all_layout == 1)
{
/* init table */
trav_table_init(&travt);
/* get the list of objects in the file */
if(h5trav_gettable(fidout, travt) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed");
for(i = 0; i < travt->nobjs; i++)
{
char *name = travt->objs[i].name;
if(travt->objs[i].type == H5TRAV_TYPE_DATASET)
{
/*-------------------------------------------------------------------------
* open
*-------------------------------------------------------------------------
*/
if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed");
if((sid = H5Dget_space(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed");
if((pid = H5Dget_create_plist(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed");
if((tid = H5Dget_type(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed");
/*-------------------------------------------------------------------------
* filter check
*-------------------------------------------------------------------------
*/
if(options->all_filter == 1)
{
if(verify_filters(pid, tid, options->n_filter_g, options->filter_g) <= 0)
ok = 0;
}
/*-------------------------------------------------------------------------
* layout check
*-------------------------------------------------------------------------
*/
if(options->all_layout == 1)
{
pack_info_t pack;
init_packobject(&pack);
pack.layout = options->layout_g;
pack.chunk = options->chunk_g;
if(verify_layout(pid, &pack) == 0)
ok = 0;
}
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if (H5Pclose(pid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
if (H5Sclose(sid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed");
if (H5Dclose(did) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
if (H5Tclose(tid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed");
} /* if */
} /* i */
/* free table */
trav_table_free(travt);
}
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if (H5Fclose(fidout) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fclose failed");
return ok;
done:
H5E_BEGIN_TRY {
H5Pclose(pid);
H5Sclose(sid);
H5Dclose(did);
H5Fclose(fidout);
if (travt)
trav_table_free(travt);
} H5E_END_TRY;
return ret_value;
} /* h5repack_verify() */
/*-------------------------------------------------------------------------
* Function: aux_assign_obj
*
* Purpose: find the object name NAME (got from the traverse list)
* in the repack options list; assign the filter information OBJ
*
* Return: 0 not found, 1 found
*
*-------------------------------------------------------------------------
*/
static
int aux_assign_obj(const char* name, /* object name from traverse list */
pack_opt_t *options, /* repack options */
pack_info_t *obj /*OUT*/) /* info about object to filter */
{
int idx, i;
pack_info_t tmp;
init_packobject(&tmp);
idx = aux_find_obj(name,options,&tmp);
/* name was on input */
if (idx>=0)
{
/* applying to all objects */
if (options->all_layout)
{
/* assign the global layout info to the OBJ info */
tmp.layout=options->layout_g;
switch (options->layout_g)
{
case H5D_CHUNKED:
tmp.chunk.rank=options->chunk_g.rank;
for ( i=0; i<tmp.chunk.rank; i++)
tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i];
break;
case H5D_LAYOUT_ERROR:
case H5D_COMPACT:
case H5D_CONTIGUOUS:
case H5D_NLAYOUTS:
break;
default:
break;
}/*switch*/
}
else
{
tmp.layout = options->op_tbl->objs[idx].layout;
switch (tmp.layout)
{
case H5D_CHUNKED:
tmp.chunk.rank = options->op_tbl->objs[idx].chunk.rank;
for ( i=0; i<tmp.chunk.rank; i++)
tmp.chunk.chunk_lengths[i]=options->op_tbl->objs[idx].chunk.chunk_lengths[i];
break;
case H5D_LAYOUT_ERROR:
case H5D_COMPACT:
case H5D_CONTIGUOUS:
case H5D_NLAYOUTS:
break;
default:
break;
}/*switch*/
}
/* applying to all objects */
if (options->all_filter)
{
/* assign the global filter */
tmp.nfilters=1;
tmp.filter[0]=options->filter_g[0];
} /* if all */
else
{
tmp.nfilters=options->op_tbl->objs[idx].nfilters;
for ( i=0; i<tmp.nfilters; i++)
{
tmp.filter[i] = options->op_tbl->objs[idx].filter[i];
}
}
} /* if idx */
/* no input name */
else
{
if (options->all_filter)
{
int k;
/* assign the global filters */
tmp.nfilters=options->n_filter_g;
for ( k = 0; k < options->n_filter_g; k++)
tmp.filter[k]=options->filter_g[k];
}
if (options->all_layout)
{
/* assign the global layout info to the OBJ info */
tmp.layout=options->layout_g;
switch (options->layout_g)
{
case H5D_CHUNKED:
tmp.chunk.rank=options->chunk_g.rank;
for ( i=0; i<tmp.chunk.rank; i++)
tmp.chunk.chunk_lengths[i]=options->chunk_g.chunk_lengths[i];
break;
case H5D_LAYOUT_ERROR:
case H5D_COMPACT:
case H5D_CONTIGUOUS:
case H5D_NLAYOUTS:
break;
default:
break;
}/*switch*/
}
}
*obj = tmp;
return 1;
}
int apply_filters(const char* name, /* object name from traverse list */
int rank, /* rank of dataset */
hsize_t *dims, /* dimensions of dataset */
size_t msize, /* size of type */
hid_t dcpl_id, /* dataset creation property list */
pack_opt_t *options, /* repack options */
int *has_filter) /* (OUT) object NAME has a filter */
{
int nfilters; /* number of filters in DCPL */
hsize_t chsize[64]; /* chunk size in elements */
H5D_layout_t layout;
int i;
pack_info_t obj;
*has_filter = 0;
if (rank==0) /* scalar dataset, do not apply */
return 0;
/*-------------------------------------------------------------------------
* initialize the assigment object
*-------------------------------------------------------------------------
*/
init_packobject(&obj);
/*-------------------------------------------------------------------------
* find options
*-------------------------------------------------------------------------
*/
if (aux_assign_obj(name,options,&obj)==0)
return 0;
/* get information about input filters */
if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
return -1;
/*-------------------------------------------------------------------------
* check if we have filters in the pipeline
* we want to replace them with the input filters
* only remove if we are inserting new ones
*-------------------------------------------------------------------------
*/
if (nfilters && obj.nfilters )
{
*has_filter = 1;
if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0)
return -1;
}
/*-------------------------------------------------------------------------
* check if there is an existent chunk
* read it only if there is not a requested layout
*-------------------------------------------------------------------------
*/
if (obj.layout == -1 )
{
if ((layout = H5Pget_layout(dcpl_id))<0)
return -1;
if (layout == H5D_CHUNKED)
{
if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0)
return -1;
obj.layout = H5D_CHUNKED;
obj.chunk.rank = rank;
for ( i = 0; i < rank; i++)
obj.chunk.chunk_lengths[i] = chsize[i];
}
}
/*-------------------------------------------------------------------------
* the type of filter and additional parameter
* type can be one of the filters
* H5Z_FILTER_NONE 0 , uncompress if compressed
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
* H5Z_FILTER_SZIP 4 , szip compression
* H5Z_FILTER_NBIT 5 , nbit compression
* H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression
*-------------------------------------------------------------------------
*/
if (obj.nfilters)
{
/*-------------------------------------------------------------------------
* filters require CHUNK layout; if we do not have one define a default
*-------------------------------------------------------------------------
*/
if (obj.layout==-1)
{
/* stripmine info */
hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */
hsize_t sm_nbytes; /*bytes per stripmine */
obj.chunk.rank = rank;
/*
* determine the strip mine size. The strip mine is
* a hyperslab whose size is manageable.
*/
sm_nbytes = msize;
for ( i = rank; i > 0; --i)
{
hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
size = 1;
sm_size[i - 1] = MIN(dims[i - 1], size);
sm_nbytes *= sm_size[i - 1];
assert(sm_nbytes > 0);
}
for ( i = 0; i < rank; i++)
{
obj.chunk.chunk_lengths[i] = sm_size[i];
}
}
for ( i=0; i<obj.nfilters; i++)
{
switch (obj.filter[i].filtn)
{
default:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_DEFLATE:
{
unsigned aggression; /* the deflate level */
aggression = obj.filter[i].cd_values[0];
/* set up for deflated data */
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if(H5Pset_deflate(dcpl_id,aggression)<0)
return -1;
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SZIP 4 , szip compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SZIP:
{
unsigned options_mask;
unsigned pixels_per_block;
options_mask = obj.filter[i].cd_values[0];
pixels_per_block = obj.filter[i].cd_values[1];
/* set up for szip data */
if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0)
return -1;
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SHUFFLE:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_shuffle(dcpl_id)<0)
return -1;
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_FLETCHER32:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_fletcher32(dcpl_id)<0)
return -1;
break;
/*----------- -------------------------------------------------------------
* H5Z_FILTER_NBIT , NBIT compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_NBIT:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_nbit(dcpl_id)<0)
return -1;
break;
/*----------- -------------------------------------------------------------
* H5Z_FILTER_SCALEOFFSET , scale+offset compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SCALEOFFSET:
{
H5Z_SO_scale_type_t scale_type;
int scale_factor;
scale_type = (H5Z_SO_scale_type_t)obj.filter[i].cd_values[0];
scale_factor = obj.filter[i].cd_values[1];
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0)
return -1;
}
break;
} /* switch */
}/*i*/
}
/*obj.nfilters*/
/*-------------------------------------------------------------------------
* layout
*-------------------------------------------------------------------------
*/
if (obj.layout>=0)
{
/* a layout was defined */
if (H5Pset_layout(dcpl_id, obj.layout)<0)
return -1;
if (H5D_CHUNKED == obj.layout)
{
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
}
else if (H5D_COMPACT == obj.layout)
{
if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY)<0)
return -1;
}
/* remove filters for the H5D_CONTIGUOUS case */
else if (H5D_CONTIGUOUS == obj.layout)
{
if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0)
return -1;
}
}
return 0;
}
int h5repack_verify(const char *fname,
pack_opt_t *options)
{
hid_t fid; /* file ID */
hid_t dset_id=-1; /* dataset ID */
hid_t dcpl_id=-1; /* dataset creation property list ID */
hid_t space_id=-1; /* space ID */
int ret=1;
unsigned int i;
int j;
trav_table_t *travt=NULL;
/* open the file */
if ((fid=H5Fopen(fname,H5F_ACC_RDONLY,H5P_DEFAULT))<0 )
return -1;
for ( i=0; i<options->op_tbl->nelems; i++)
{
char* name=options->op_tbl->objs[i].path;
pack_info_t *obj = &options->op_tbl->objs[i];
/*-------------------------------------------------------------------------
* open
*-------------------------------------------------------------------------
*/
if ((dset_id=H5Dopen(fid,name))<0)
goto error;
if ((space_id=H5Dget_space(dset_id))<0)
goto error;
if ((dcpl_id=H5Dget_create_plist(dset_id))<0)
goto error;
/*-------------------------------------------------------------------------
* filter check
*-------------------------------------------------------------------------
*/
for ( j=0; j<obj->nfilters; j++)
{
if (has_filter(dcpl_id,obj->filter[j].filtn)==0)
ret=0;
}
/*-------------------------------------------------------------------------
* layout check
*-------------------------------------------------------------------------
*/
if ((obj->layout!=-1) && (has_layout(dcpl_id,obj)==0))
ret=0;
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if (H5Pclose(dcpl_id)<0)
goto error;
if (H5Sclose(space_id)<0)
goto error;
if (H5Dclose(dset_id)<0)
goto error;
}
/*-------------------------------------------------------------------------
* check for the "all" objects option
*-------------------------------------------------------------------------
*/
if (options->all_filter==1 || options->all_layout==1)
{
/* init table */
trav_table_init(&travt);
/* get the list of objects in the file */
if (h5trav_gettable(fid,travt)<0)
goto error;
for ( i=0; i<travt->nobjs; i++)
{
char* name=travt->objs[i].name;
switch ( travt->objs[i].type )
{
case H5G_DATASET:
/*-------------------------------------------------------------------------
* open
*-------------------------------------------------------------------------
*/
if ((dset_id=H5Dopen(fid,name))<0)
goto error;
if ((space_id=H5Dget_space(dset_id))<0)
goto error;
if ((dcpl_id=H5Dget_create_plist(dset_id))<0)
goto error;
/*-------------------------------------------------------------------------
* filter check
*-------------------------------------------------------------------------
*/
if (options->all_filter==1 ){
if (has_filter(dcpl_id,options->filter_g.filtn)==0)
ret=0;
}
/*-------------------------------------------------------------------------
* layout check
*-------------------------------------------------------------------------
*/
if (options->all_layout==1){
pack_info_t pack;
init_packobject(&pack);
pack.layout=options->layout_g;
pack.chunk=options->chunk_g;
if (has_layout(dcpl_id,&pack)==0)
ret=0;
}
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if (H5Pclose(dcpl_id)<0)
goto error;
if (H5Sclose(space_id)<0)
goto error;
if (H5Dclose(dset_id)<0)
goto error;
break;
default:
break;
} /* switch */
} /* i */
/* free table */
trav_table_free(travt);
}
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if (H5Fclose(fid)<0)
return -1;
return ret;
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl_id);
H5Sclose(space_id);
H5Dclose(dset_id);
H5Fclose(fid);
if (travt)
trav_table_free(travt);
} H5E_END_TRY;
return -1;
}
