/* Test with chunked and compressed SDS. This routine creates
* "ChunkedDeflateData" and "ChunkedNoDeflateData" SDSs and writes the same
* data to both. It will then use SDgetdatasize to verify the sizes.
*/
static intn test_chkcmp_SDSs(int32 fid)
{
int32 sds_id, sds_index;
int32 cmpsds_id, cmpsds_index;
int32 flag, maxcache, new_maxcache;
int32 dim_sizes[RANK], origin[RANK];
HDF_CHUNK_DEF c_def; /* Chunking definitions */
int32 fill_value = 0; /* Fill value */
int32 comp_size1=0, uncomp_size1=0;
int32 comp_size2=0, uncomp_size2=0;
intn status;
int num_errs = 0; /* number of errors so far */
/* Declare chunks data type and initialize some of them. */
int16 chunk1[CHK_X][CHK_Y] = { {1, 1},
{1, 1},
{1, 1} };
int16 chunk3[CHK_X][CHK_Y] = { {3, 3},
{3, 3},
{3, 3} };
int32 chunk2[CHK_X][CHK_Y] = { {2, 2},
{2, 2},
{2, 2} };
/* Initialize chunk size */
HDmemset(&c_def, 0, sizeof(c_def)) ;
c_def.chunk_lengths[0] = CHK_X;
c_def.chunk_lengths[1] = CHK_Y;
/* Create Y_LENGTH2 x X_LENGTH2 SDS */
dim_sizes[0] = Y_LENGTH2;
dim_sizes[1] = X_LENGTH2;
cmpsds_id = SDcreate(fid, "ChunkedDeflateData", DFNT_INT32, RANK, dim_sizes);
CHECK(cmpsds_id, FAIL, "test_chkcmp_SDSs: SDcreate 'ChunkedDeflateData'");
sds_id = SDcreate(fid, "ChunkedNoDeflateData", DFNT_INT32, RANK, dim_sizes);
CHECK(sds_id, FAIL, "test_chkcmp_SDSs: SDcreate 'ChunkedNoDeflateData'");
/* Fill the SDS array with the fill value */
status = SDsetfillvalue(cmpsds_id, (VOIDP)&fill_value);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDsetfillvalue 'ChunkedDeflateData'");
status = SDsetfillvalue(sds_id, (VOIDP)&fill_value);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDsetfillvalue 'ChunkedNoDeflateData'");
/* Set info for chunking and compression */
flag = HDF_CHUNK | HDF_COMP;
c_def.comp.comp_type = COMP_CODE_DEFLATE;
c_def.comp.cinfo.deflate.level = 6;
status = SDsetchunk(cmpsds_id, c_def, flag);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDsetchunk 'ChunkedDeflateData'");
/* Set info for chunking and compression */
HDmemset(&c_def, 0, sizeof(c_def)) ;
c_def.chunk_lengths[0] = CHK_X;
c_def.chunk_lengths[1] = CHK_Y;
flag = HDF_CHUNK;
status = SDsetchunk(sds_id, c_def, flag);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDsetchunk 'ChunkedNoDeflateData'");
/* Set chunk cache to hold maximum of 3 chunks */
maxcache = 3;
flag = 0;
new_maxcache = SDsetchunkcache(cmpsds_id, maxcache, flag);
CHECK(new_maxcache, FAIL, "test_chkcmp_SDSs: SDsetchunkcache 'ChunkedDeflateData'");
new_maxcache = SDsetchunkcache(sds_id, maxcache, flag);
CHECK(new_maxcache, FAIL, "test_chkcmp_SDSs: SDsetchunkcache 'ChunkedNoDeflateData'");
/* Terminate access to the dataset before writing data to it. */
status = SDendaccess(cmpsds_id);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDendaccess 'ChunkedDeflateData'");
status = SDendaccess(sds_id);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDendaccess 'ChunkedNoDeflateData'");
/* Check that this SDS is still empty after the call to SDsetchunk */
check_datasizes(fid, "ChunkedDeflateData", 0, 0, &num_errs);
check_datasizes(fid, "ChunkedNoDeflateData", 0, 0, &num_errs);
/* Re-select the datasets, write chunks using SDwritechunk function, then
check their data sizes */
/* Get index of dataset using its name */
cmpsds_index = SDnametoindex(fid, "ChunkedDeflateData");
CHECK(cmpsds_index, FAIL, "test_chkcmp_SDSs: SDnametoindex 'ChunkedDeflateData'");
sds_index = SDnametoindex(fid, "ChunkedNoDeflateData");
CHECK(sds_index, FAIL, "test_chkcmp_SDSs: SDnametoindex 'ChunkedNoDeflateData'");
/* Select the datasets for access */
cmpsds_id = SDselect(fid, cmpsds_index);
CHECK(cmpsds_id, FAIL, "test_chkcmp_SDSs: SDselect 'ChunkedDeflateData'");
sds_id = SDselect(fid, sds_index);
CHECK(cmpsds_id, FAIL, "test_chkcmp_SDSs: SDselect 'ChunkedNoDeflateData'");
/* Write the chunk with the coordinates (0,0) */
origin[0] = 0;
origin[1] = 0;
status = SDwritechunk(cmpsds_id, origin, (VOIDP) chunk1);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDwritechunk 'ChunkedDeflateData'");
status = SDwritechunk(sds_id, origin, (VOIDP) chunk1);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDwritechunk 'ChunkedNoDeflateData'");
/* Write the chunk with the coordinates (1,0) */
origin[0] = 1;
origin[1] = 0;
status = SDwritechunk(cmpsds_id, origin, (VOIDP) chunk3);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDwritechunk 'ChunkedDeflateData'");
status = SDwritechunk(sds_id, origin, (VOIDP) chunk3);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDwritechunk 'ChunkedNoDeflateData'");
/* Write the chunk with the coordinates (0,1) */
origin[0] = 0;
origin[1] = 1;
status = SDwritechunk(cmpsds_id, origin, (VOIDP) chunk2);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDwritechunk 'ChunkedDeflateData'");
status = SDwritechunk(sds_id, origin, (VOIDP) chunk2);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDwritechunk 'ChunkedNoDeflateData'");
/* Terminate access to the datasets */
status = SDendaccess(cmpsds_id);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDendaccess 'ChunkedDeflateData'");
status = SDendaccess(sds_id);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDendaccess 'ChunkedNoDeflateData'");
/* Verify the compressed and non-compressed data sizes of the datasets */
/* Open dataset 'ChunkedDeflateData' */
cmpsds_id = SDselect(fid, cmpsds_index);
CHECK(cmpsds_id, FAIL, "test_chkcmp_SDSs: SDselect 'ChunkedDeflateData'");
/* Get the data sizes */
status = SDgetdatasize(cmpsds_id, &comp_size1, &uncomp_size1);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDgetdatasize 'ChunkedDeflateData'");
/* Open dataset 'ChunkedNoDeflateData' */
sds_id = SDselect(fid, sds_index);
CHECK(sds_id, FAIL, "test_chkcmp_SDSs: SDselect 'ChunkedNoDeflateData'");
/* Get the data sizes */
status = SDgetdatasize(sds_id, &comp_size2, &uncomp_size2);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDgetdatasize 'ChunkedNoDeflateData'");
/* Non-compressed data sizes of the two datasets should be the same */
VERIFY(uncomp_size1, uncomp_size2, "test_chkcmp_SDSs: non-compressed data sizes might be incorrect");
/* In this test, compressed data size should be smaller than non-compressed
data size */
if (comp_size1 >= uncomp_size1)
{
printf("*** Routine test_chkcmp_SDSs: FAILED at line %d ***\n", __LINE__);
printf(" In this test, compressed data size (%d) should be smaller than non-compressed data size (%d)\n", comp_size1, uncomp_size1);
num_errs++;
}
/* Terminate access to the data sets. */
status = SDendaccess(sds_id);
CHECK(sds_id, FAIL, "test_chkcmp_SDSs: SDendaccess 'ChunkedNoDeflateData'");
status = SDendaccess(cmpsds_id);
CHECK(sds_id, FAIL, "test_chkcmp_SDSs: SDendaccess 'ChunkedDeflateData'");
/* Return the number of errors that's been kept track of so far */
return num_errs;
} /* test_chkcmp_SDSs */
/*
* Verify that object headers are held in the cache until they are linked
* to a location in the graph, or assigned an ID. This is done by
* creating an object header, then forcing it out of the cache by creating
* local heaps until the object header is evicted from the cache, then
* modifying the object header. The refcount on the object header is
* checked as verifying that the object header has remained in the cache.
*/
static herr_t
test_ohdr_cache(char *filename, hid_t fapl)
{
hid_t file = -1; /* File ID */
hid_t my_fapl; /* FAPL ID */
hid_t my_dxpl; /* DXPL ID */
H5AC_cache_config_t mdc_config; /* Metadata cache configuration info */
H5F_t *f = NULL; /* File handle */
H5HL_t *lheap, *lheap2, *lheap3; /* Pointer to local heaps */
haddr_t lheap_addr, lheap_addr2, lheap_addr3; /* Local heap addresses */
H5O_loc_t oh_loc; /* Object header location */
time_t time_new; /* Time value for modification time message */
unsigned rc; /* Refcount for object */
TESTING("object header creation in cache");
/* Make a copy of the FAPL */
if((my_fapl = H5Pcopy(fapl)) < 0)
FAIL_STACK_ERROR
/* Tweak down the size of the metadata cache to only 64K */
mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
if(H5Pget_mdc_config(my_fapl, &mdc_config) < 0)
FAIL_STACK_ERROR
mdc_config.set_initial_size = TRUE;
mdc_config.initial_size = 32 * 1024;
mdc_config.max_size = 64 * 1024;
mdc_config.min_size = 8 * 1024;
if(H5Pset_mdc_config(my_fapl, &mdc_config) < 0)
FAIL_STACK_ERROR
/* Make a copy of the default DXPL */
if((my_dxpl = H5Pcopy(H5AC_ind_read_dxpl_id)) < 0)
FAIL_STACK_ERROR
/* Create the file to operate on */
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0)
FAIL_STACK_ERROR
if(H5Pclose(my_fapl) < 0)
FAIL_STACK_ERROR
if(NULL == (f = (H5F_t *)H5I_object(file)))
FAIL_STACK_ERROR
if(H5AC_ignore_tags(f) < 0)
FAIL_STACK_ERROR
/* Create object (local heap) that occupies most of cache */
if(H5HL_create(f, my_dxpl, (31 * 1024), &lheap_addr) < 0)
FAIL_STACK_ERROR
/* Protect local heap (which actually pins it in the cache) */
if(NULL == (lheap = H5HL_protect(f, my_dxpl, lheap_addr, H5AC__READ_ONLY_FLAG)))
FAIL_STACK_ERROR
/* Create an object header */
HDmemset(&oh_loc, 0, sizeof(oh_loc));
if(H5O_create(f, my_dxpl, (size_t)2048, (size_t)1, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0)
FAIL_STACK_ERROR
/* Query object header information */
rc = 0;
if(H5O_get_rc(&oh_loc, my_dxpl, &rc) < 0)
FAIL_STACK_ERROR
if(0 != rc)
TEST_ERROR
/* Create object (local heap) that occupies most of cache */
if(H5HL_create(f, my_dxpl, (31 * 1024), &lheap_addr2) < 0)
FAIL_STACK_ERROR
/* Protect local heap (which actually pins it in the cache) */
if(NULL == (lheap2 = H5HL_protect(f, my_dxpl, lheap_addr2, H5AC__READ_ONLY_FLAG)))
FAIL_STACK_ERROR
/* Unprotect local heap (which actually unpins it from the cache) */
if(H5HL_unprotect(lheap2) < 0)
FAIL_STACK_ERROR
/* Create object header message in new object header */
time_new = 11111111;
if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, my_dxpl) < 0)
FAIL_STACK_ERROR
/* Create object (local heap) that occupies most of cache */
if(H5HL_create(f, my_dxpl, (31 * 1024), &lheap_addr3) < 0)
FAIL_STACK_ERROR
/* Protect local heap (which actually pins it in the cache) */
if(NULL == (lheap3 = H5HL_protect(f, my_dxpl, lheap_addr3, H5AC__READ_ONLY_FLAG)))
FAIL_STACK_ERROR
/* Unprotect local heap (which actually unpins it from the cache) */
if(H5HL_unprotect(lheap3) < 0)
FAIL_STACK_ERROR
/* Query object header information */
/* (Note that this is somewhat of a weak test, since it doesn't actually
* verify that the object header was evicted from the cache, but it's
* very difficult to verify when an entry is evicted from the cache in
* a non-invasive way -QAK)
*/
rc = 0;
if(H5O_get_rc(&oh_loc, my_dxpl, &rc) < 0)
FAIL_STACK_ERROR
if(0 != rc)
TEST_ERROR
/* Decrement reference count o object header */
if(H5O_dec_rc_by_loc(&oh_loc, my_dxpl) < 0)
FAIL_STACK_ERROR
/* Close object header created */
if(H5O_close(&oh_loc, NULL) < 0)
FAIL_STACK_ERROR
/* Unprotect local heap (which actually unpins it from the cache) */
if(H5HL_unprotect(lheap) < 0)
FAIL_STACK_ERROR
if(H5Pclose(my_dxpl) < 0)
FAIL_STACK_ERROR
if(H5Fclose(file) < 0)
FAIL_STACK_ERROR
PASSED();
return SUCCEED;
error:
H5E_BEGIN_TRY {
H5Fclose(file);
} H5E_END_TRY;
return FAIL;
} /* test_ohdr_cache() */
/*-------------------------------------------------------------------------
* test_compress
*
* Ensures that a FL packet table can be compressed.
* This test creates a file named TEST_COMPRESS_FILE
*
*-------------------------------------------------------------------------
*/
static int
test_compress(void)
{
hid_t fid1 = -1;
herr_t err;
hid_t table = -1;
hid_t part_t = -1;
hid_t dset_id = -1;
hid_t plist_id = -1;
size_t c;
size_t num_elems = 1;
unsigned filter_vals[1];
particle_t readPart[1];
hsize_t count;
TESTING("packet table compression");
/* Create a file. */
if((fid1 = H5Fcreate(TEST_COMPRESS_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create a datatype for the particle struct */
part_t = make_particle_type();
HDassert(part_t != -1);
/* Create a new table with compression level 8 */
table = H5PTcreate_fl(fid1, "Compressed Test Dataset", part_t, (hsize_t)80, 8);
if( H5PTis_valid(table) < 0) TEST_ERROR;
/* We can now use this table exactly the same way we use a normal uncompressed
* packet table, and it should pass the same tests. */
/* Add many particles */
for(c = 0; c < BIG_TABLE_SIZE ; c+=8)
{
/* Append eight particles at once*/
err = H5PTappend(table, (size_t)8, &(testPart[0]));
if( err < 0) TEST_ERROR;
}
/* Count the number of packets in the table */
err = H5PTget_num_packets(table, &count);
if( err < 0) TEST_ERROR;
if( count != BIG_TABLE_SIZE ) TEST_ERROR;
/* Read particles to ensure that all of them were written correctly */
HDmemset(readPart, 0, sizeof(readPart));
for(c = 0; c < BIG_TABLE_SIZE; c++)
{
err = H5PTget_next(table, 1, readPart);
if(err < 0) TEST_ERROR;
/* Ensure that particles were read correctly */
if( cmp_par(c % 8, 0, testPart, readPart) != 0) TEST_ERROR;
}
/* Close the table */
err = H5PTclose(table);
if( err < 0) TEST_ERROR;
/* Open the packet table as a regular dataset and make sure that the
* compression filter is set.
*/
dset_id = H5Dopen2(fid1, "Compressed Test Dataset", H5P_DEFAULT);
if( dset_id < 0) TEST_ERROR;
plist_id = H5Dget_create_plist(dset_id);
if( plist_id < 0) TEST_ERROR;
err = H5Pget_filter_by_id2(plist_id, H5Z_FILTER_DEFLATE, NULL, &num_elems,
filter_vals, 0, NULL, NULL);
if( err < 0) TEST_ERROR;
/* The compression level should be 8, the value we passed in */
if(filter_vals[0] != 8) TEST_ERROR;
/* Clean up */
err = H5Pclose(plist_id);
if( err < 0) TEST_ERROR;
err = H5Dclose(dset_id);
if( err < 0) TEST_ERROR;
/* Create a new table without compression. */
table = H5PTcreate_fl(fid1, "Uncompressed Dataset", part_t, (hsize_t)80, -1);
if(table < 0) TEST_ERROR;
/* Close the packet table */
err = H5PTclose(table);
if( err < 0) TEST_ERROR;
/* Open the packet table as a regular dataset and make sure that the
* compression filter is not set.
*/
dset_id = H5Dopen2(fid1, "Uncompressed Dataset", H5P_DEFAULT);
if( dset_id < 0) TEST_ERROR;
plist_id = H5Dget_create_plist(dset_id);
if( plist_id < 0) TEST_ERROR;
H5E_BEGIN_TRY {
err = H5Pget_filter_by_id2(plist_id, H5Z_FILTER_DEFLATE, NULL, &num_elems,
filter_vals, 0, NULL, NULL);
if( err >= 0) TEST_ERROR;
} H5E_END_TRY
/* Clean up */
err = H5Pclose(plist_id);
if( err < 0) TEST_ERROR;
err = H5Dclose(dset_id);
if( err < 0) TEST_ERROR;
/* Close the datatype and the file */
err = H5Tclose(part_t);
if( err < 0) TEST_ERROR;
err = H5Fclose(fid1);
if( err < 0) TEST_ERROR;
PASSED();
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(plist_id);
H5Dclose(dset_id);
H5Tclose(part_t);
H5PTclose(table);
H5Fclose(fid1);
} H5E_END_TRY
H5_FAILED();
return -1;
}
/*-------------------------------------------------------------------------
* Function: main
*
* Modifications:
* 2/2010; Vailin Choi
* Get the size of user block
*
*-------------------------------------------------------------------------
*/
int
main(int argc, const char *argv[])
{
iter_t iter;
const char *fname = NULL;
hid_t fid = -1;
hid_t fcpl;
struct handler_t *hand = NULL;
H5F_info_t finfo;
int i;
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
/* Disable error reporting */
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
/* Initialize h5tools lib */
h5tools_init();
if((hand = parse_command_line(argc, argv))==NULL) {
goto done;
}
fname = argv[opt_ind];
printf("Filename: %s\n", fname);
HDmemset(&iter, 0, sizeof(iter));
fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
if(fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
goto done;
} /* end if */
/* Initialize iter structure */
iter.fid = fid;
if(H5Fget_filesize(fid, &iter.filesize) < 0)
warn_msg("Unable to retrieve file size\n");
HDassert(iter.filesize != 0);
/* Get storge info for file-level structures */
if(H5Fget_info(fid, &finfo) < 0)
warn_msg("Unable to retrieve file info\n");
else {
iter.super_ext_size = finfo.super_ext_size;
iter.SM_hdr_storage_size = finfo.sohm.hdr_size;
iter.SM_index_storage_size = finfo.sohm.msgs_info.index_size;
iter.SM_heap_storage_size = finfo.sohm.msgs_info.heap_size;
} /* end else */
if((fcpl = H5Fget_create_plist(fid)) < 0)
warn_msg("Unable to retrieve file creation property\n");
if(H5Pget_userblock(fcpl, &iter.ublk_size) < 0)
warn_msg("Unable to retrieve userblock size\n");
/* Walk the objects or all file */
if(display_object) {
unsigned u;
u = 0;
while(hand[u].obj) {
if (h5trav_visit(fid, hand[u].obj, TRUE, TRUE, obj_stats, lnk_stats, &iter) < 0)
warn_msg("Unable to traverse object \"%s\"\n", hand[u].obj);
else
print_statistics(hand[u].obj, &iter);
u++;
} /* end while */
} /* end if */
else {
if (h5trav_visit(fid, "/", TRUE, TRUE, obj_stats, lnk_stats, &iter) < 0)
warn_msg("Unable to traverse objects/links in file \"%s\"\n", fname);
else
print_statistics("/", &iter);
} /* end else */
done:
if(hand) {
for (i = 0; i < argc; i++)
if(hand[i].obj) {
free(hand[i].obj);
hand[i].obj=NULL;
}
free(hand);
hand = NULL;
/* Free iter structure */
iter_free(&iter);
if(fid >= 0 && H5Fclose(fid) < 0) {
error_msg("unable to close file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
}
}
leave(h5tools_getstatus());
}
/*--------------------------------------------------------------------------
NAME
H5O_attr_debug
PURPOSE
Prints debugging information for an attribute message
USAGE
void *H5O_attr_debug(f, mesg, stream, indent, fwidth)
H5F_t *f; IN: pointer to the HDF5 file struct
const void *mesg; IN: Pointer to the source attribute struct
FILE *stream; IN: Pointer to the stream for output data
int indent; IN: Amount to indent information by
int fwidth; IN: Field width (?)
RETURNS
Non-negative on success/Negative on failure
DESCRIPTION
This function prints debugging output to the stream passed as a
parameter.
--------------------------------------------------------------------------*/
static herr_t
H5O_attr_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE * stream, int indent,
int fwidth)
{
const H5A_t *mesg = (const H5A_t *)_mesg;
H5O_shared_t sh_mesg; /* Shared message information */
void *dt_mesg; /* Pointer to datatype message to dump */
herr_t (*debug)(H5F_t*, hid_t, const void*, FILE*, int, int)=NULL;
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5O_attr_debug);
/* check args */
assert(f);
assert(stream);
assert(indent >= 0);
assert(fwidth >= 0);
fprintf(stream, "%*s%-*s \"%s\"\n", indent, "", fwidth,
"Name:",
mesg->name);
fprintf(stream, "%*s%-*s %u\n", indent, "", fwidth,
"Initialized:",
(unsigned int)mesg->initialized);
fprintf(stream, "%*s%-*s %u\n", indent, "", fwidth,
"Opened:",
(unsigned int)mesg->ent_opened);
fprintf(stream, "%*sSymbol table entry...\n", indent, "");
H5G_ent_debug(f, dxpl_id, &(mesg->ent), stream, indent+3, MAX(0, fwidth-3),
HADDR_UNDEF);
fprintf(stream, "%*sData type...\n", indent, "");
fprintf(stream, "%*s%-*s %lu\n", indent+3, "", MAX(0,fwidth-3),
"Size:",
(unsigned long)(mesg->dt_size));
fprintf (stream, "%*s%-*s %s\n", indent+3, "", MAX(0,fwidth-3),
"Shared:",
(H5T_committed(mesg->dt) ? "Yes" : "No")
);
if(H5T_committed(mesg->dt)) {
/* Reset shared message information */
HDmemset(&sh_mesg,0,sizeof(H5O_shared_t));
/* Get shared message information from datatype */
if ((H5O_MSG_DTYPE->get_share)(f, mesg->dt, &sh_mesg/*out*/)<0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTENCODE, FAIL, "can't retrieve shared message information");
debug=H5O_MSG_SHARED->debug;
dt_mesg=&sh_mesg;
} /* end if */
else {
debug=H5O_MSG_DTYPE->debug;
dt_mesg=mesg->dt;
} /* end else */
if(debug)
(debug)(f, dxpl_id, dt_mesg, stream, indent+3, MAX(0, fwidth-3));
else
fprintf(stream, "%*s<No info for this message>\n", indent + 6, "");
fprintf(stream, "%*sData space...\n", indent, "");
fprintf(stream, "%*s%-*s %lu\n", indent+3, "", MAX(0,fwidth-3),
"Size:",
(unsigned long)(mesg->ds_size));
H5S_debug(f, dxpl_id, mesg->ds, stream, indent+3, MAX(0, fwidth-3));
done:
FUNC_LEAVE_NOAPI(ret_value);
}
/********************************************************************
Name: test_getszipdata() - verifies that SZIP compressed data can be read
when either SZIP library encoder or only decoder is present
Description:
This test function opens the existing file "sds_szipped.dat" that
contains a dataset with SZIP compression and verifies that the SZIP
compressed data can be read with or without the encoder as long as
the szlib is available.
The input file, sds_szipped.dat, is generated by the program
mfhdf/libsrc/gen_sds_szipped.c
Return value:
The number of errors occurred in this routine.
BMR - Oct 10, 2008
*********************************************************************/
#ifdef H4_HAVE_LIBSZ /* needed to read data, either decoder or encoder */
static intn test_getszipdata()
{
/************************* Variable declaration **************************/
int32 sd_id, sds_id;
intn status;
int32 dim_sizes[2], array_rank, num_type, attributes;
char name[H4_MAX_NC_NAME];
comp_info c_info;
int32 start[2], edges[2];
int16 fill_value = 0; /* Fill value */
int i,j;
int num_errs = 0; /* number of errors so far */
int32 out_data[SZ_LENGTH][SZ_WIDTH];
char testfile[512] = "";
char *basename = "sds_szipped.dat";
/* data to compare against read data from sds_szipped.dat */
int32 in_data[SZ_LENGTH][SZ_WIDTH]={
100,100,200,200,300,
0, 0, 0, 0, 0,
100,100,200,200,300,
400,300,200,100,0,
300,300, 0,400,300,
300,300, 0,400,300,
300,300, 0,400,300,
0, 0,600,600,300,
500,500,600,600,300,
0, 0,600,600,300,
0, 0,600,600,300,
0, 0,600,600,300,
0, 0,600,600,300,
500,500,600,600,300,
500,500,600,600,300,
500,500,600,600,300 };
/********************* End of variable declaration ***********************/
/* Make the name for the test file */
make_datafilename(basename, testfile, sizeof(testfile));
/* Open the file */
sd_id = SDstart (testfile, DFACC_READ);
CHECK(sd_id, FAIL, "SDstart");
/* Get the first SDS */
sds_id = SDselect (sd_id, 0);
CHECK(sds_id, FAIL, "SDselect:Failed to select a data set for szip compression testing");
/* Retrieve information of the data set */
status = SDgetinfo(sds_id, name, &array_rank, dim_sizes, &num_type, &attributes);
CHECK(status, FAIL, "SDgetinfo");
/* Prepare for reading */
start[0] = 0;
start[1] = 0;
edges[0] = dim_sizes[0];
edges[1] = dim_sizes[1];
/* Wipe out the output buffer */
HDmemset(&out_data, 0, sizeof(out_data));
/* Read the data set */
status = SDreaddata (sds_id, start, NULL, edges, (VOIDP)out_data);
CHECK(status, FAIL, "SDreaddata");
/* Compare read data against input data */
for (j=0; j<SZ_LENGTH; j++)
{
for (i=0; i<SZ_WIDTH; i++)
if (out_data[j][i] != in_data[j][i])
{
fprintf(stderr,"This one: Bogus val in loc [%d][%d] in compressed dset, want %ld got %ld\n", j, i, (long)in_data[j][i], (long)out_data[j][i]);
num_errs++;
}
}
/* Terminate access to the data set */
status = SDendaccess (sds_id);
CHECK(status, FAIL, "SDendaccess");
/* Terminate access to the SD interface and close the file */
status = SDend (sd_id);
CHECK(status, FAIL, "SDend");
/* Return the number of errors that's been kept track of so far */
return num_errs;
} /* test_getszipdata */
static intn test_szip_chunk()
{
/************************* Variable declaration **************************/
int32 sd_id, sds_id, sds_index;
intn status;
int32 flag, maxcache, new_maxcache;
int32 dim_sizes[2], origin[2];
HDF_CHUNK_DEF c_def; /* Chunking definitions */
HDF_CHUNK_DEF c_def_out; /* Chunking definitions */
int32 c_flags, c_flags_out;
int32 all_data[LENGTH_CH][WIDTH_CH];
int32 start[2], edges[2];
int32 comp_size=0, uncomp_size=0;
int32 chunk_out[CLENGTH][CWIDTH];
int32 row[CWIDTH] = { 5, 5 };
int32 column[CLENGTH] = { 4, 4, 4 };
int32 fill_value = 0; /* Fill value */
comp_coder_t comp_type; /* to retrieve compression type into */
comp_info cinfo; /* compression information structure */
int num_errs = 0; /* number of errors so far */
int i,j;
/*
* Define all chunks. Note that chunks 4 & 5 are not used to write,
* only to verify the read data. The 'row' and 'column' are used
* to write in the place of these chunks.
*/
int32 chunk1[CLENGTH][CWIDTH] = { 1, 1,
1, 1,
1, 1 };
int32 chunk2[CLENGTH][CWIDTH] = { 2, 2,
2, 2,
2, 2 };
int32 chunk3[CLENGTH][CWIDTH] = { 3, 3,
3, 3,
3, 3 };
int32 chunk4[CLENGTH][CWIDTH] = { 0, 4,
0, 4,
0, 4 };
int32 chunk5[CLENGTH][CWIDTH] = { 0, 0,
5, 5,
0, 0 };
int32 chunk6[CLENGTH][CWIDTH] = { 6, 6,
6, 6,
6, 6 };
/* Initialize chunk lengths. */
c_def.comp.chunk_lengths[0] = CLENGTH;
c_def.comp.chunk_lengths[1] = CWIDTH;
/* Create the file and initialize SD interface. */
sd_id = SDstart (FILE_NAME, DFACC_CREATE);
CHECK(sd_id, FAIL, "SDstart");
/* Create LENGTH_CHxWIDTH_CH SDS. */
dim_sizes[0] = LENGTH_CH;
dim_sizes[1] = WIDTH_CH;
sds_id = SDcreate (sd_id, SDS_NAME_CH,DFNT_INT32, RANK_CH, dim_sizes);
CHECK(sds_id, FAIL, "SDcreate:Failed to create a data set for chunking/szip compression testing");
/* Fill the SDS array with the fill value. */
status = SDsetfillvalue (sds_id, (VOIDP)&fill_value);
CHECK(status, FAIL, "SDsetfillvalue");
/* Set parameters for Chunking/SZIP */
c_def.comp.comp_type = COMP_CODE_SZIP;
c_def.comp.cinfo.szip.pixels_per_block = 2;
c_def.comp.cinfo.szip.options_mask = SZ_EC_OPTION_MASK;
c_def.comp.cinfo.szip.options_mask |= SZ_MSB_OPTION_MASK;
c_def.comp.cinfo.szip.bits_per_pixel = 0;
c_def.comp.cinfo.szip.pixels = 0;
c_def.comp.cinfo.szip.pixels_per_scanline = 0;
c_flags = HDF_CHUNK | HDF_COMP;
status = SDsetchunk (sds_id, c_def, c_flags);
CHECK(status, FAIL, "SDsetchunk");
/* Set chunk cache to hold maximum of 3 chunks. */
maxcache = 3;
flag = 0;
new_maxcache = SDsetchunkcache (sds_id, maxcache, flag);
CHECK(new_maxcache, FAIL, "SDsetchunkcache");
HDmemset(&c_def_out, 0, sizeof(HDF_CHUNK_DEF));
c_flags_out = 0;
status = SDgetchunkinfo(sds_id, &c_def_out, &c_flags_out);
CHECK(status, FAIL, "SDgetchunkinfo");
VERIFY(c_flags_out, c_flags, "SDgetchunkinfo");
VERIFY(c_def_out.comp.comp_type, COMP_CODE_SZIP, "SDgetchunkinfo");
/*
* Write chunks using SDwritechunk function. Chunks can be written
* in any order.
*/
/* Write the chunk with the coordinates (0,0). */
origin[0] = 0;
origin[1] = 0;
status = SDwritechunk (sds_id, origin, (VOIDP) chunk1);
CHECK(status, FAIL, "SDwritechunk");
/* Write the chunk with the coordinates (1,0). */
origin[0] = 1;
origin[1] = 0;
status = SDwritechunk (sds_id, origin, (VOIDP) chunk3);
CHECK(status, FAIL, "SDwritechunk");
/* Write the chunk with the coordinates (0,1). */
origin[0] = 0;
origin[1] = 1;
status = SDwritechunk (sds_id, origin, (VOIDP) chunk2);
CHECK(status, FAIL, "SDwritechunk");
/* Write chunk with the coordinates (1,2) using SDwritedata function. */
start[0] = 6;
start[1] = 2;
edges[0] = 3;
edges[1] = 2;
status = SDwritedata (sds_id, start, NULL, edges, (VOIDP) chunk6);
CHECK(status, FAIL, "SDwritedata");
/* Fill second column in the chunk with the coordinates (1,1) using
* SDwritedata function. */
start[0] = 3;
start[1] = 3;
edges[0] = 3;
edges[1] = 1;
status = SDwritedata (sds_id, start, NULL, edges, (VOIDP) column);
CHECK(status, FAIL, "SDwritedata");
/* Fill second row in the chunk with the coordinates (0,2) using
* SDwritedata function. */
start[0] = 7;
start[1] = 0;
edges[0] = 1;
edges[1] = 2;
status = SDwritedata (sds_id, start, NULL, edges, (VOIDP) row);
CHECK(status, FAIL, "SDwritedata");
/* Terminate access to the data set. */
status = SDendaccess (sds_id);
CHECK(status, FAIL, "SDendaccess");
/* Terminate access to the SD interface and close the file. */
status = SDend (sd_id);
CHECK(status, FAIL, "SDend");
/*
* Verify the compressed data
*/
/* Reopen the file and access the first data set. */
sd_id = SDstart (FILE_NAME, DFACC_READ);
sds_index = 0;
sds_id = SDselect (sd_id, sds_index);
CHECK(sds_id, FAIL, "SDselect:Failed to select a data set for chunking/szip compression testing");
/* Retrieve compression information about the dataset */
comp_type = COMP_CODE_INVALID; /* reset variables before retrieving info */
HDmemset(&cinfo, 0, sizeof(cinfo)) ;
status = SDgetcompinfo(sds_id, &comp_type, &cinfo);
CHECK(status, FAIL, "SDgetcompinfo");
VERIFY(comp_type, COMP_CODE_SZIP, "SDgetcompinfo");
/* Retrieve compression method alone from the dataset */
comp_type = COMP_CODE_INVALID; /* reset variables before retrieving info */
status = SDgetcomptype(sds_id, &comp_type);
CHECK(status, FAIL, "SDgetcomptype");
VERIFY(comp_type, COMP_CODE_SZIP, "SDgetcomptype");
/* Read the entire data set using SDreaddata function. */
start[0] = 0;
start[1] = 0;
edges[0] = LENGTH_CH;
edges[1] = WIDTH_CH;
status = SDreaddata (sds_id, start, NULL, edges, (VOIDP)all_data);
CHECK(status, FAIL, "SDreaddata");
/*
* This is how the entire array should look like:
*
* 1 1 2 2
* 1 1 2 2
* 1 1 2 2
* 3 3 0 4
* 3 3 0 4
* 3 3 0 4
* 0 0 6 6
* 5 5 6 6
* 0 0 6 6
*/
/* Read chunk #4 with the coordinates (1,1) and verify it. */
origin[0] = 1;
origin[1] = 1;
status = SDreadchunk (sds_id, origin, chunk_out);
CHECK(status, FAIL, "SDreadchunk");
for (j=0; j<CLENGTH; j++)
{
for (i=0; i<CWIDTH; i++)
{
if (chunk_out[j][i] != chunk4[j][i])
{
fprintf(stderr,"Bogus val in loc [%d][%d] in chunk #4, want %ld got %ld\n", j, i, chunk4[j][i], chunk_out[j][i]);
num_errs++;
}
}
}
/*
* Read chunk #5 with the coordinates (2,0) and verify it.
*/
origin[0] = 2;
origin[1] = 0;
status = SDreadchunk (sds_id, origin, chunk_out);
CHECK(status, FAIL, "SDreadchunk");
for (j=0; j<CLENGTH; j++)
{
for (i=0; i<CWIDTH; i++)
if (chunk_out[j][i] != chunk5[j][i])
{
fprintf(stderr,"Bogus val in loc [%d][%d] in chunk #5, want %ld got %ld\n", j, i, chunk5[j][i], chunk_out[j][i]);
num_errs++;
}
}
/* Get the data sizes */
status = SDgetdatasize(sds_id, &comp_size, &uncomp_size);
CHECK(status, FAIL, "test_chkcmp_SDSs: SDgetdatasize");
/* Terminate access to the data set. */
status = SDendaccess (sds_id);
CHECK(status, FAIL, "SDendaccess");
/* Terminate access to the SD interface and close the file. */
status = SDend (sd_id);
CHECK(status, FAIL, "SDend");
/* Return the number of errors that's been kept track of so far */
return num_errs;
} /* test_szip_chunk */
/*-------------------------------------------------------------------------
* Function: H5MF_init_merge_flags
*
* Purpose: Initialize the free space section+aggregator merge flags
* for the file.
*
* Return: SUCCEED/FAIL
*
* Programmer: Quincey Koziol
* Friday, February 1, 2008
*
*-------------------------------------------------------------------------
*/
herr_t
H5MF_init_merge_flags(H5F_t *f)
{
H5MF_aggr_merge_t mapping_type; /* Type of free list mapping */
H5FD_mem_t type; /* Memory type for iteration */
hbool_t all_same; /* Whether all the types map to the same value */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* check args */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->lf);
/* Iterate over all the free space types to determine if sections of that type
* can merge with the metadata or small 'raw' data aggregator
*/
all_same = TRUE;
for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
/* Check for any different type mappings */
if(f->shared->fs_type_map[type] != f->shared->fs_type_map[H5FD_MEM_DEFAULT]) {
all_same = FALSE;
break;
} /* end if */
/* Check for all allocation types mapping to the same free list type */
if(all_same) {
if(f->shared->fs_type_map[H5FD_MEM_DEFAULT] == H5FD_MEM_DEFAULT)
mapping_type = H5MF_AGGR_MERGE_SEPARATE;
else
mapping_type = H5MF_AGGR_MERGE_TOGETHER;
} /* end if */
else {
/* Check for raw data mapping into same list as metadata */
if(f->shared->fs_type_map[H5FD_MEM_DRAW] == f->shared->fs_type_map[H5FD_MEM_SUPER])
mapping_type = H5MF_AGGR_MERGE_SEPARATE;
else {
hbool_t all_metadata_same; /* Whether all metadata go in same free list */
/* One or more allocation type don't map to the same free list type */
/* Check if all the metadata allocation types map to the same type */
all_metadata_same = TRUE;
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
/* Skip checking raw data free list mapping */
/* (global heap is treated as raw data) */
if(type != H5FD_MEM_DRAW && type != H5FD_MEM_GHEAP) {
/* Check for any different type mappings */
if(f->shared->fs_type_map[type] != f->shared->fs_type_map[H5FD_MEM_SUPER]) {
all_metadata_same = FALSE;
break;
} /* end if */
} /* end if */
/* Check for all metadata on same free list */
if(all_metadata_same)
mapping_type = H5MF_AGGR_MERGE_DICHOTOMY;
else
mapping_type = H5MF_AGGR_MERGE_SEPARATE;
} /* end else */
} /* end else */
/* Based on mapping type, initialize merging flags for each free list type */
switch(mapping_type) {
case H5MF_AGGR_MERGE_SEPARATE:
/* Don't merge any metadata together */
HDmemset(f->shared->fs_aggr_merge, 0, sizeof(f->shared->fs_aggr_merge));
/* Check if merging raw data should be allowed */
/* (treat global heaps as raw data) */
if(H5FD_MEM_DRAW == f->shared->fs_type_map[H5FD_MEM_DRAW] ||
H5FD_MEM_DEFAULT == f->shared->fs_type_map[H5FD_MEM_DRAW]) {
f->shared->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA;
f->shared->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA;
} /* end if */
break;
case H5MF_AGGR_MERGE_DICHOTOMY:
/* Merge all metadata together (but not raw data) */
HDmemset(f->shared->fs_aggr_merge, H5F_FS_MERGE_METADATA, sizeof(f->shared->fs_aggr_merge));
/* Allow merging raw data allocations together */
/* (treat global heaps as raw data) */
f->shared->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA;
f->shared->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA;
break;
case H5MF_AGGR_MERGE_TOGETHER:
/* Merge all allocation types together */
HDmemset(f->shared->fs_aggr_merge, (H5F_FS_MERGE_METADATA | H5F_FS_MERGE_RAWDATA), sizeof(f->shared->fs_aggr_merge));
break;
default:
HGOTO_ERROR(H5E_RESOURCE, H5E_BADVALUE, FAIL, "invalid mapping type")
} /* end switch */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5MF_init_merge_flags() */
/*-------------------------------------------------------------------------
* Function: test_find
*
* Purpose: Test bit find operations. This is just the basic stuff; more
* rigorous testing will be performed by the other test
* functions.
*
* Return: Success: 0
*
* Failure: -1
*
* Programmer: Robb Matzke
* Tuesday, June 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
test_find (void)
{
uint8_t v1[8];
int i;
ssize_t n;
TESTING("bit search operations");
/* The zero length buffer */
HDmemset(v1, 0xaa, sizeof v1);
n = H5T_bit_find(v1, (size_t)0, (size_t)0, H5T_BIT_LSB, TRUE);
if(-1 != n) {
H5_FAILED();
puts (" Zero length test failed (lsb)!");
goto failed;
}
n = H5T_bit_find(v1, (size_t)0, (size_t)0, H5T_BIT_MSB, TRUE);
if(-1 != n) {
H5_FAILED();
puts (" Zero length test failed (msb)!");
goto failed;
}
/* The zero buffer */
HDmemset(v1, 0, sizeof v1);
n = H5T_bit_find(v1, (size_t)0, 8 * sizeof(v1), H5T_BIT_LSB, TRUE);
if(-1 != n) {
H5_FAILED();
puts (" Zero buffer test failed (lsb)!");
goto failed;
}
n = H5T_bit_find(v1, (size_t)0, 8 * sizeof(v1), H5T_BIT_MSB, TRUE);
if(-1 != n) {
H5_FAILED();
puts (" Zero buffer test failed (msb)!");
goto failed;
}
/* Try all combinations of one byte */
for(i = 0; i < 8 * (int)sizeof(v1); i++) {
HDmemset(v1, 0, sizeof v1);
v1[i / 8] = 1 << (i % 8);
n = H5T_bit_find(v1, (size_t)0, 8 * sizeof(v1), H5T_BIT_LSB, TRUE);
if((ssize_t)i != n) {
H5_FAILED();
printf (" Test for set bit %d failed (lsb)!\n", i);
goto failed;
}
n = H5T_bit_find(v1, (size_t)0, 8 * sizeof(v1), H5T_BIT_MSB, TRUE);
if((ssize_t)i != n) {
H5_FAILED();
printf (" Test for set bit %d failed (msb)!\n", i);
goto failed;
}
}
/* The one buffer */
HDmemset(v1, 0xff, sizeof v1);
n = H5T_bit_find(v1, (size_t)0, 8 * sizeof(v1), H5T_BIT_LSB, FALSE);
if(-1 != n) {
H5_FAILED();
puts (" One buffer test failed (lsb)!");
goto failed;
}
n = H5T_bit_find(v1, (size_t)0, 8 * sizeof(v1), H5T_BIT_MSB, FALSE);
if(-1 != n) {
H5_FAILED();
puts (" One buffer test failed (msb)!");
goto failed;
}
/* Try all combinations of one byte */
for (i=0; i<8*(int)sizeof(v1); i++) {
memset (v1, 0xff, sizeof v1);
v1[i/8] &= ~(1<<(i%8));
n = H5T_bit_find (v1, (size_t)0, 8*sizeof(v1), H5T_BIT_LSB, FALSE);
if ((ssize_t)i!=n) {
H5_FAILED();
printf (" Test for clear bit %d failed (lsb)!\n", i);
goto failed;
}
n = H5T_bit_find (v1, (size_t)0, 8*sizeof(v1), H5T_BIT_MSB, FALSE);
if ((ssize_t)i!=n) {
H5_FAILED();
printf (" Test for clear bit %d failed (lsb)!\n", i);
goto failed;
}
}
PASSED();
return 0;
failed:
printf (" v = 0x");
for (i=0; i<(int)sizeof(v1); i++) printf ("%02x", v1[i]);
printf ("\n");
return -1;
}
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose: Create group(s) in an HDF5 file
*
* Programmer: Quincey Koziol, 2/13/2007
*
*-------------------------------------------------------------------------
*/
int
main(int argc, const char *argv[])
{
hid_t fid; /* HDF5 file ID */
hid_t fapl_id; /* File access property list ID */
hid_t lcpl_id; /* Link creation property list ID */
size_t curr_group; /* Current group to create */
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
/* Disable the HDF5 library's error reporting */
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
/* Initialize h5tools lib */
h5tools_init();
/* Parse command line */
HDmemset(¶ms, 0, sizeof(params));
if(parse_command_line(argc, argv, ¶ms) < 0) {
error_msg("unable to parse command line arguments\n");
leave(EXIT_FAILURE);
} /* end if */
/* Create file access property list */
if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) {
error_msg("Could not create file access property list\n");
leave(EXIT_FAILURE);
} /* end if */
/* Check for creating groups with new format version */
if(params.latest) {
/* Set the "use the latest version of the format" bounds */
if(H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) {
error_msg("Could not set property for using latest version of the format\n");
leave(EXIT_FAILURE);
} /* end if */
/* Display some output if requested */
if(params.verbose)
printf("%s: Creating groups with latest version of the format\n", h5tools_getprogname());
} /* end if */
/* Attempt to open an existing HDF5 file first */
fid = h5tools_fopen(params.fname, H5F_ACC_RDWR, fapl_id, NULL, NULL, 0);
/* If we couldn't open an existing file, try creating file */
/* (use "EXCL" instead of "TRUNC", so we don't blow away existing non-HDF5 file) */
if(fid < 0)
fid = H5Fcreate(params.fname, H5F_ACC_EXCL, H5P_DEFAULT, fapl_id);
/* Test for error in opening file */
if(fid < 0) {
error_msg("Could not open output file '%s'\n", params.fname);
leave(EXIT_FAILURE);
} /* end if */
/* Create link creation property list */
if((lcpl_id = H5Pcreate(H5P_LINK_CREATE)) < 0) {
error_msg("Could not create link creation property list\n");
leave(EXIT_FAILURE);
} /* end if */
/* Check for creating intermediate groups */
if(params.parents) {
/* Set the intermediate group creation property */
if(H5Pset_create_intermediate_group(lcpl_id, TRUE) < 0) {
error_msg("Could not set property for creating parent groups\n");
leave(EXIT_FAILURE);
} /* end if */
/* Display some output if requested */
if(params.verbose)
printf("%s: Creating parent groups\n", h5tools_getprogname());
} /* end if */
/* Loop over creating requested groups */
for(curr_group = 0; curr_group < params.ngroups; curr_group++) {
hid_t gid; /* Group ID */
/* Attempt to create a group */
if((gid = H5Gcreate2(fid, params.groups[curr_group], lcpl_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
error_msg("Could not create group '%s'\n", params.groups[curr_group]);
leave(EXIT_FAILURE);
} /* end if */
/* Close the group */
if(H5Gclose(gid) < 0) {
error_msg("Could not close group '%s'??\n", params.groups[curr_group]);
leave(EXIT_FAILURE);
} /* end if */
/* Display some output if requested */
if(params.verbose)
printf("%s: created group '%s'\n", h5tools_getprogname(), params.groups[curr_group]);
} /* end for */
/* Close link creation property list */
if(H5Pclose(lcpl_id) < 0) {
error_msg("Could not close link creation property list\n");
leave(EXIT_FAILURE);
} /* end if */
/* Close file */
if(H5Fclose(fid) < 0) {
error_msg("Could not close output file '%s'??\n", params.fname);
leave(EXIT_FAILURE);
} /* end if */
/* Close file access property list */
if(H5Pclose(fapl_id) < 0) {
error_msg("Could not close file access property list\n");
leave(EXIT_FAILURE);
} /* end if */
/* Shut down h5tools lib */
h5tools_close();
leave(EXIT_SUCCESS);
} /* end main() */
/*-------------------------------------------------------------------------
* Function: test_logging_api
*
* Purpose: Tests the API calls that affect mdc logging
*
* Return: Success: 0
* Failure: -1
*-------------------------------------------------------------------------
*/
static herr_t
test_logging_api(void)
{
hid_t fapl = -1;
hbool_t is_enabled;
hbool_t is_enabled_out;
hbool_t start_on_access;
hbool_t start_on_access_out;
char *location = NULL;
size_t size;
hid_t fid;
hid_t gid;
hbool_t is_currently_logging;
char group_name[8];
char filename[1024];
int i;
TESTING("metadata cache log api calls");
fapl = h5_fileaccess();
h5_fixname(FILE_NAME, fapl, filename, sizeof filename);
/* Set up metadata cache logging */
is_enabled = TRUE;
start_on_access = FALSE;
if(H5Pset_mdc_log_options(fapl, is_enabled, LOG_LOCATION, start_on_access) < 0)
TEST_ERROR;
/* Check to make sure that the property list getter returns the correct
* location string buffer size;
*/
is_enabled_out = FALSE;
start_on_access_out = TRUE;
location = NULL;
size = 999;
if(H5Pget_mdc_log_options(fapl, &is_enabled_out, location, &size,
&start_on_access_out) < 0)
TEST_ERROR;
if(size != strlen(LOG_LOCATION) + 1)
TEST_ERROR;
/* Check to make sure that the property list getter works */
if(NULL == (location = (char *)HDcalloc(size, sizeof(char))))
TEST_ERROR;
if(H5Pget_mdc_log_options(fapl, &is_enabled_out, location, &size,
&start_on_access_out) < 0)
TEST_ERROR;
if((is_enabled != is_enabled_out)
|| (start_on_access != start_on_access_out)
|| HDstrcmp(LOG_LOCATION, location))
TEST_ERROR;
/* Create a file */
if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
TEST_ERROR;
if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if(H5Pclose(fapl) < 0)
TEST_ERROR;
/* Check to see if the logging flags were set correctly */
is_enabled = FALSE;
is_currently_logging = TRUE;
if((H5Fget_mdc_logging_status(fid, &is_enabled, &is_currently_logging) < 0)
|| (is_enabled != TRUE)
|| (is_currently_logging != FALSE))
TEST_ERROR;
/* Turn on logging and check flags */
if(H5Fstart_mdc_logging(fid) < 0)
TEST_ERROR;
is_enabled = FALSE;
is_currently_logging = FALSE;
if((H5Fget_mdc_logging_status(fid, &is_enabled, &is_currently_logging) < 0)
|| (is_enabled != TRUE)
|| (is_currently_logging != TRUE))
TEST_ERROR;
/* Perform some manipulations */
for(i = 0; i < N_GROUPS; i++) {
HDmemset(group_name, 0, 8);
HDsnprintf(group_name, 8, "%d", i);
if((gid = H5Gcreate2(fid, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
if(H5Gclose(gid) < 0)
TEST_ERROR;
}
/* Turn off logging and check flags */
if(H5Fstop_mdc_logging(fid) < 0)
TEST_ERROR;
is_enabled = FALSE;
is_currently_logging = TRUE;
if((H5Fget_mdc_logging_status(fid, &is_enabled, &is_currently_logging) < 0)
|| (is_enabled != TRUE)
|| (is_currently_logging != FALSE))
TEST_ERROR;
/* Clean up */
HDfree(location);
if(H5Fclose(fid) < 0)
TEST_ERROR;
PASSED();
return 0;
error:
return 1;
} /* test_logging_api() */
/*
* This creates a dataset serially with chunks, each of CHUNK_SIZE
* elements. The allocation time is set to H5D_ALLOC_TIME_EARLY. Another
* routine will open this in parallel for extension test.
*/
static void
create_chunked_dataset(const char *filename, int chunk_factor, write_type write_pattern)
{
hid_t file_id, dataset; /* handles */
hid_t dataspace,memspace;
hid_t cparms;
hsize_t dims[1];
hsize_t maxdims[1] = {H5S_UNLIMITED};
hsize_t chunk_dims[1] ={CHUNK_SIZE};
hsize_t count[1];
hsize_t stride[1];
hsize_t block[1];
hsize_t offset[1]; /* Selection offset within dataspace */
/* Variables used in reading data back */
char buffer[CHUNK_SIZE];
long nchunks;
herr_t hrc;
MPI_Offset filesize, /* actual file size */
est_filesize; /* estimated file size */
/* set up MPI parameters */
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
/* Only MAINPROCESS should create the file. Others just wait. */
if (MAINPROCESS){
nchunks=chunk_factor*mpi_size;
dims[0]=nchunks*CHUNK_SIZE;
/* Create the data space with unlimited dimensions. */
dataspace = H5Screate_simple (1, dims, maxdims);
VRFY((dataspace >= 0), "");
memspace = H5Screate_simple(1, chunk_dims, NULL);
VRFY((memspace >= 0), "");
/* Create a new file. If file exists its contents will be overwritten. */
file_id = H5Fcreate(h5_rmprefix(filename), H5F_ACC_TRUNC, H5P_DEFAULT,
H5P_DEFAULT);
VRFY((file_id >= 0), "H5Fcreate");
/* Modify dataset creation properties, i.e. enable chunking */
cparms = H5Pcreate(H5P_DATASET_CREATE);
VRFY((cparms >= 0), "");
hrc = H5Pset_alloc_time(cparms, H5D_ALLOC_TIME_EARLY);
VRFY((hrc >= 0), "");
hrc = H5Pset_chunk(cparms, 1, chunk_dims);
VRFY((hrc >= 0), "");
/* Create a new dataset within the file using cparms creation properties. */
dataset = H5Dcreate2(file_id, DSET_NAME, H5T_NATIVE_UCHAR, dataspace, H5P_DEFAULT, cparms, H5P_DEFAULT);
VRFY((dataset >= 0), "");
if(write_pattern == sec_last) {
HDmemset(buffer, 100, CHUNK_SIZE);
count[0] = 1;
stride[0] = 1;
block[0] = chunk_dims[0];
offset[0] = (nchunks-2)*chunk_dims[0];
hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block);
VRFY((hrc >= 0), "");
/* Write sec_last chunk */
hrc = H5Dwrite(dataset, H5T_NATIVE_UCHAR, memspace, dataspace, H5P_DEFAULT, buffer);
VRFY((hrc >= 0), "H5Dwrite");
} /* end if */
/* Close resources */
hrc = H5Dclose (dataset);
VRFY((hrc >= 0), "");
dataset = -1;
hrc = H5Sclose (dataspace);
VRFY((hrc >= 0), "");
hrc = H5Sclose (memspace);
VRFY((hrc >= 0), "");
hrc = H5Pclose (cparms);
VRFY((hrc >= 0), "");
hrc = H5Fclose (file_id);
VRFY((hrc >= 0), "");
file_id = -1;
/* verify file size */
filesize = get_filesize(filename);
est_filesize = nchunks * CHUNK_SIZE * sizeof(unsigned char);
VRFY((filesize >= est_filesize), "file size check");
}
/* Make sure all processes are done before exiting this routine. Otherwise,
* other tests may start and change the test data file before some processes
* of this test are still accessing the file.
*/
MPI_Barrier(MPI_COMM_WORLD);
}
/*-------------------------------------------------------------------------
* Function: read_records
*
* Purpose: For a given dataset, checks to make sure that the stated
* and actual sizes are the same. If they are not, then
* we have an inconsistent dataset due to a SWMR error.
*
* Parameters: const char *filename
* The SWMR test file's name.
*
* unsigned verbose
* Whether verbose console output is desired.
*
* unsigned long nrecords
* The total number of records to read.
*
* unsigned poll_time
* The amount of time to sleep (s).
*
* unsigned reopen_count
*
*
* Return: Success: 0
* Failure: -1
*
*-------------------------------------------------------------------------
*/
static int
read_records(const char *filename, unsigned verbose, unsigned long nrecords,
unsigned poll_time, unsigned reopen_count)
{
hid_t fid; /* File ID */
hid_t aid; /* Attribute ID */
time_t start_time; /* Starting time */
hid_t mem_sid; /* Memory dataspace ID */
symbol_t record; /* The record to add to the dataset */
unsigned seed; /* Seed for random number generator */
unsigned iter_to_reopen = reopen_count; /* # of iterations until reopen */
unsigned long u; /* Local index variable */
hid_t fapl;
HDassert(filename);
HDassert(poll_time != 0);
/* Create file access property list */
if((fapl = h5_fileaccess()) < 0)
return -1;
H5Pset_fclose_degree(fapl, H5F_CLOSE_SEMI);
/* Emit informational message */
if(verbose)
HDfprintf(stderr, "Opening file: %s\n", filename);
/* Open the file */
if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, fapl)) < 0)
return -1;
/* Seed the random number generator with the attribute in the file */
if((aid = H5Aopen(fid, "seed", H5P_DEFAULT)) < 0)
return -1;
if(H5Aread(aid, H5T_NATIVE_UINT, &seed) < 0)
return -1;
if(H5Aclose(aid) < 0)
return -1;
HDsrandom(seed);
/* Reset the record */
/* (record's 'info' field might need to change for each record written, also) */
HDmemset(&record, 0, sizeof(record));
/* Create a dataspace for the record to read */
if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
return -1;
/* Emit informational message */
if(verbose)
HDfprintf(stderr, "Reading records\n");
/* Get the starting time */
start_time = HDtime(NULL);
/* Read records */
for(u = 0; u < nrecords; u++) {
symbol_info_t *symbol = NULL; /* Symbol (dataset) */
htri_t attr_exists; /* Whether the sequence number attribute exists */
unsigned long file_u; /* Attribute sequence number (writer's "u") */
/* Get a random dataset, according to the symbol distribution */
symbol = choose_dataset();
/* Fill in "nrecords" field. Note that this depends on the writer
* using the same algorithm and "nrecords" */
symbol->nrecords = nrecords / 5;
/* Wait until we can read the dataset */
do {
/* Check if sequence attribute exists */
if((attr_exists = H5Aexists_by_name(fid, symbol->name, "seq", H5P_DEFAULT)) < 0)
return -1;
if(attr_exists) {
/* Read sequence number attribute */
if((aid = H5Aopen_by_name(fid, symbol->name, "seq", H5P_DEFAULT, H5P_DEFAULT)) < 0)
return -1;
if(H5Aread(aid, H5T_NATIVE_ULONG, &file_u) < 0)
return -1;
if(H5Aclose(aid) < 0)
return -1;
/* Check if sequence number is at least u - if so, this should
* guarantee that this record has been written */
if(file_u >= u)
break;
} /* end if */
/* Check for timeout */
if(HDtime(NULL) >= (time_t)(start_time + (time_t)TIMEOUT)) {
HDfprintf(stderr, "Reader timed out\n");
return -1;
} /* end if */
/* Pause */
HDsleep(poll_time);
/* Retrieve and print the collection of metadata read retries */
if(print_metadata_retries_info(fid) < 0)
HDfprintf(stderr, "Warning: could not obtain metadata retries info\n");
/* Reopen the file */
if(H5Fclose(fid) < 0)
return -1;
if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, fapl)) < 0)
return -1;
iter_to_reopen = reopen_count;
} while(1);
/* Emit informational message */
if(verbose)
HDfprintf(stderr, "Checking dataset %lu\n", u);
/* Check dataset */
if(check_dataset(fid, verbose, symbol, &record, mem_sid) < 0)
return -1;
HDmemset(&record, 0, sizeof(record));
/* Check for reopen */
iter_to_reopen--;
if(iter_to_reopen == 0) {
/* Emit informational message */
if(verbose)
HDfprintf(stderr, "Reopening file: %s\n", filename);
/* Retrieve and print the collection of metadata read retries */
if(print_metadata_retries_info(fid) < 0)
HDfprintf(stderr, "Warning: could not obtain metadata retries info\n");
/* Reopen the file */
if(H5Fclose(fid) < 0)
return -1;
if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, fapl)) < 0)
return -1;
iter_to_reopen = reopen_count;
} /* end if */
} /* end while */
/* Retrieve and print the collection of metadata read retries */
if(print_metadata_retries_info(fid) < 0)
HDfprintf(stderr, "Warning: could not obtain metadata retries info\n");
/* Close file */
if(H5Fclose(fid) < 0)
return -1;
/* Close the memory dataspace */
if(H5Sclose(mem_sid) < 0)
return -1;
return 0;
} /* end read_records() */
/* Test SDSs with unlimited dimensions. This routine creates SDSs with
unlimited dimensions, writes data to it, and checks the sizes returned
by SDgetdatasize
*/
static intn test_extend_SDSs(int32 fid)
{
int32 sds_id, sds_index;
int32 dimsize[2], start[2], edges[2];
int32 dimsize1[1], start1[1], edges1[1];
int32 data[Y_LENGTH][X_LENGTH];
float fdata[Y_LENGTH];
int32 output[Y_LENGTH][X_LENGTH];
intn status;
int i, j;
int num_errs = 0; /* number of errors so far */
/* Initialize data for the dataset */
for (j = 0; j < Y_LENGTH; j++) {
for (i = 0; i < X_LENGTH; i++)
data[j][i] = (i + j) + 1;
}
/* Create a 2x2 dataset called "EmptyDataset" */
dimsize[0] = SD_UNLIMITED;
dimsize[1] = X_LENGTH;
sds_id = SDcreate(fid, "AppendableDataset 1", DFNT_INT32, 2, dimsize);
CHECK(sds_id, FAIL, "test_extend_SDSs: SDcreate 'AppendableDataset 1'");
/* Write the stored data to the dataset */
start[0] = start[1] = 0;
edges[0] = Y_LENGTH;
edges[1] = X_LENGTH;
status = SDwritedata(sds_id, start, NULL, edges, (VOIDP)data);
CHECK(sds_id, FAIL, "test_extend_SDSs: SDwritedata");
/* Check data. */
HDmemset(&output, 0, sizeof(output));
status = SDreaddata(sds_id, start, NULL, edges, (VOIDP)output);
CHECK(sds_id, FAIL, "test_extend_SDSs: SDreaddata");
/* Initialize data for the dataset */
for (j = 0; j < Y_LENGTH; j++)
for (i = 0; i < X_LENGTH; i++)
if (output[j][i] != data[j][i])
fprintf(stderr, "Read value (%d) differs from written (%d) at [%d,%d]\n", output[j][i], data[j][i], j, i);
/* Close this SDS */
status = SDendaccess(sds_id);
CHECK(status, FAIL, "test_extend_SDSs: SDendaccess");
/* Check that this SDS is empty */
check_datasizes(fid, "AppendableDataset 1", Y_LENGTH*X_LENGTH*SIZE_INT32, Y_LENGTH*X_LENGTH*SIZE_INT32, &num_errs);
/* Create another dataset with 1 unlimited dimension */
sds_id = SDcreate(fid, "AppendableDataset 2", DFNT_FLOAT64, 1, dimsize);
CHECK(sds_id, FAIL, "test_extend_SDSs: SDcreate 'AppendableDataset 2'");
/* Define the location and size of the data to be written to the dataset */
start1[0] = 0;
edges1[0] = Y_LENGTH;
/* Write the stored data to the dataset */
status = SDwritedata(sds_id, start1, NULL, edges1, (VOIDP)fdata);
CHECK(sds_id, FAIL, "test_extend_SDSs: SDwritedata");
/* Close this SDS */
status = SDendaccess(sds_id);
CHECK(status, FAIL, "test_extend_SDSs: SDendaccess");
/* Check the size of the data of this SDS */
check_datasizes(fid, "AppendableDataset 2", Y_LENGTH*SIZE_FLOAT64, Y_LENGTH*SIZE_FLOAT64, &num_errs);
/* Return the number of errors that's been kept track of so far */
return num_errs;
} /* test_extend_SDSs */
/*------------------------------------------------------------------------- * Function: diff_datasetid * * Purpose: check for comparable datasets and read into a compatible * memory type * * Return: Number of differences found * * Programmer: Pedro Vicente, [email protected] * * Date: May 9, 2003 * * Modifications: * * * October 2006: Read by hyperslabs for big datasets. * * A threshold of H5TOOLS_MALLOCSIZE (128 MB) is the limit upon which I/O hyperslab is done * i.e., if the memory needed to read a dataset is greater than this limit, * then hyperslab I/O is done instead of one operation I/O * For each dataset, the memory needed is calculated according to * * memory needed = number of elements * size of each element * * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations * are done * * H5Dread( input_dataset1 ) * H5Dread( input_dataset2 ) * * with all elements in the datasets selected. If the memory needed is greater than * H5TOOLS_MALLOCSIZE, then the following operations are done instead: * * a strip mine is defined for each dimension k (a strip mine is defined as a * hyperslab whose size is memory manageable) according to the formula * * (1) strip_mine_size[k ] = MIN(dimension[k ], H5TOOLS_BUFSIZE / size of memory type) * * where H5TOOLS_BUFSIZE is a constant currently defined as 1MB. This formula assures * that for small datasets (small relative to the H5TOOLS_BUFSIZE constant), the strip * mine size k is simply defined as its dimension k, but for larger datasets the * hyperslab size is still memory manageable. * a cycle is done until the number of elements in the dataset is reached. In each * iteration, two parameters are defined for the function H5Sselect_hyperslab, * the start and size of each hyperslab, according to * * (2) hyperslab_size [k] = MIN(dimension[k] - hyperslab_offset[k], strip_mine_size [k]) * * where hyperslab_offset [k] is initially set to zero, and later incremented in * hyperslab_size[k] offsets. The reason for the operation * * dimension[k] - hyperslab_offset[k] * * in (2) is that, when using the strip mine size, it assures that the "remaining" part * of the dataset that does not fill an entire strip mine is processed. * *------------------------------------------------------------------------- */ hsize_t diff_datasetid( hid_t did1, hid_t did2, const char *obj1_name, const char *obj2_name, diff_opt_t *options) { hid_t sid1=-1; hid_t sid2=-1; hid_t f_tid1=-1; hid_t f_tid2=-1; hid_t m_tid1=-1; hid_t m_tid2=-1; size_t m_size1; size_t m_size2; H5T_sign_t sign1; H5T_sign_t sign2; int rank1; int rank2; hsize_t nelmts1; hsize_t nelmts2; hsize_t dims1[H5S_MAX_RANK]; hsize_t dims2[H5S_MAX_RANK]; hsize_t maxdim1[H5S_MAX_RANK]; hsize_t maxdim2[H5S_MAX_RANK]; const char *name1=NULL; /* relative names */ const char *name2=NULL; hsize_t storage_size1; hsize_t storage_size2; hsize_t nfound=0; /* number of differences found */ int can_compare=1; /* do diff or not */ void *buf1=NULL; void *buf2=NULL; void *sm_buf1=NULL; void *sm_buf2=NULL; hid_t sm_space; /*stripmine data space */ size_t need; /* bytes needed for malloc */ int i; unsigned int vl_data = 0; /*contains VL datatypes */ h5difftrace("diff_datasetid start\n"); /* Get the dataspace handle */ if ( (sid1 = H5Dget_space(did1)) < 0 ) goto error; /* Get rank */ if ( (rank1 = H5Sget_simple_extent_ndims(sid1)) < 0 ) goto error; /* Get the dataspace handle */ if ( (sid2 = H5Dget_space(did2)) < 0 ) goto error; /* Get rank */ if ( (rank2 = H5Sget_simple_extent_ndims(sid2)) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid1,dims1,maxdim1) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid2,dims2,maxdim2) < 0 ) { goto error; } /*------------------------------------------------------------------------- * get the file data type *------------------------------------------------------------------------- */ /* Get the data type */ if ( (f_tid1 = H5Dget_type(did1)) < 0 ) goto error; /* Get the data type */ if ( (f_tid2 = H5Dget_type(did2)) < 0 ) { goto error; } /*------------------------------------------------------------------------- * check for empty datasets *------------------------------------------------------------------------- */ h5difftrace("check for empty datasets\n"); storage_size1=H5Dget_storage_size(did1); storage_size2=H5Dget_storage_size(did2); if (storage_size1==0 || storage_size2==0) { if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) parallel_print("Not comparable: <%s> or <%s> is an empty dataset\n", obj1_name, obj2_name); can_compare=0; options->not_cmp=1; } /*------------------------------------------------------------------------- * check for comparable TYPE and SPACE *------------------------------------------------------------------------- */ if (diff_can_type(f_tid1, f_tid2, rank1, rank2, dims1, dims2, maxdim1, maxdim2, obj1_name, obj2_name, options, 0)!=1) { can_compare=0; } /*------------------------------------------------------------------------- * memory type and sizes *------------------------------------------------------------------------- */ h5difftrace("check for memory type and sizes\n"); if ((m_tid1=h5tools_get_native_type(f_tid1)) < 0) goto error; if ((m_tid2=h5tools_get_native_type(f_tid2)) < 0) goto error; m_size1 = H5Tget_size( m_tid1 ); m_size2 = H5Tget_size( m_tid2 ); /*------------------------------------------------------------------------- * check for different signed/unsigned types *------------------------------------------------------------------------- */ if (can_compare) { h5difftrace("can_compare for sign\n"); sign1=H5Tget_sign(m_tid1); sign2=H5Tget_sign(m_tid2); if ( sign1 != sign2 ) { h5difftrace("sign1 != sign2\n"); if ((options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) { parallel_print("Not comparable: <%s> has sign %s ", obj1_name, get_sign(sign1)); parallel_print("and <%s> has sign %s\n", obj2_name, get_sign(sign2)); } can_compare=0; options->not_cmp=1; } } /* Check if type is either VLEN-data or VLEN-string to reclaim any * VLEN memory buffer later */ if( TRUE == h5tools_detect_vlen(m_tid1) ) vl_data = TRUE; /*------------------------------------------------------------------------ * only attempt to compare if possible *------------------------------------------------------------------------- */ if(can_compare) /* it is possible to compare */ { h5difftrace("can_compare attempt\n"); /*----------------------------------------------------------------- * get number of elements *------------------------------------------------------------------ */ nelmts1 = 1; for(i = 0; i < rank1; i++) nelmts1 *= dims1[i]; nelmts2 = 1; for(i = 0; i < rank2; i++) nelmts2 *= dims2[i]; HDassert(nelmts1 == nelmts2); /*----------------------------------------------------------------- * "upgrade" the smaller memory size *------------------------------------------------------------------ */ h5difftrace("upgrade the smaller memory size?\n"); if (FAIL == match_up_memsize (f_tid1, f_tid2, &m_tid1, &m_tid2, &m_size1, &m_size2)) goto error; /* print names */ if(obj1_name) name1 = diff_basename(obj1_name); if(obj2_name) name2 = diff_basename(obj2_name); /*---------------------------------------------------------------- * read/compare *----------------------------------------------------------------- */ need = (size_t)(nelmts1 * m_size1); /* bytes needed */ if(need < H5TOOLS_MALLOCSIZE) { buf1 = HDmalloc(need); buf2 = HDmalloc(need); } /* end if */ if(buf1 != NULL && buf2 != NULL) { h5difftrace("buf1 != NULL && buf2 != NULL\n"); if(H5Dread(did1, m_tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf1) < 0) goto error; if(H5Dread(did2, m_tid2, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf2) < 0) goto error; /* array diff */ nfound = diff_array(buf1, buf2, nelmts1, (hsize_t)0, rank1, dims1, options, name1, name2, m_tid1, did1, did2); /* reclaim any VL memory, if necessary */ if(vl_data) { H5Dvlen_reclaim(m_tid1, sid1, H5P_DEFAULT, buf1); H5Dvlen_reclaim(m_tid2, sid2, H5P_DEFAULT, buf2); } /* end if */ } /* end if */ else /* possibly not enough memory, read/compare by hyperslabs */ { size_t p_type_nbytes = m_size1; /*size of memory type */ hsize_t p_nelmts = nelmts1; /*total selected elmts */ hsize_t elmtno; /*counter */ int carry; /*counter carry value */ /* stripmine info */ hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ hsize_t sm_nbytes; /*bytes per stripmine */ hsize_t sm_nelmts; /*elements per stripmine*/ /* hyperslab info */ hsize_t hs_offset[H5S_MAX_RANK]; /*starting offset */ hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ hsize_t hs_nelmts; /*elements in request */ hsize_t zero[8]; /*vector of zeros */ /* * determine the strip mine size and allocate a buffer. The strip mine is * a hyperslab whose size is manageable. */ sm_nbytes = p_type_nbytes; for(i = rank1; i > 0; --i) { hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; if(size == 0) /* datum size > H5TOOLS_BUFSIZE */ size = 1; sm_size[i - 1] = MIN(dims1[i - 1], size); sm_nbytes *= sm_size[i - 1]; HDassert(sm_nbytes > 0); } /* end for */ /* malloc return code should be verified. * If fail, need to handle the error. * This else branch should be recoded as a separate function. * Note that there are many "goto error" within this branch * that fails to address freeing other objects created here. * E.g., sm_space. */ sm_buf1 = HDmalloc((size_t)sm_nbytes); HDassert(sm_buf1); sm_buf2 = HDmalloc((size_t)sm_nbytes); HDassert(sm_buf2); sm_nelmts = sm_nbytes / p_type_nbytes; sm_space = H5Screate_simple(1, &sm_nelmts, NULL); /* the stripmine loop */ HDmemset(hs_offset, 0, sizeof hs_offset); HDmemset(zero, 0, sizeof zero); for(elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) { /* calculate the hyperslab size */ if(rank1 > 0) { for(i = 0, hs_nelmts = 1; i < rank1; i++) { hs_size[i] = MIN(dims1[i] - hs_offset[i], sm_size[i]); hs_nelmts *= hs_size[i]; } /* end for */ if(H5Sselect_hyperslab(sid1, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if(H5Sselect_hyperslab(sid2, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if(H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL) < 0) goto error; } /* end if */ else hs_nelmts = 1; if(H5Dread(did1,m_tid1,sm_space,sid1,H5P_DEFAULT,sm_buf1) < 0) goto error; if(H5Dread(did2,m_tid2,sm_space,sid2,H5P_DEFAULT,sm_buf2) < 0) goto error; /* get array differences. in the case of hyperslab read, increment the number of differences found in each hyperslab and pass the position at the beggining for printing */ nfound += diff_array(sm_buf1, sm_buf2, hs_nelmts, elmtno, rank1, dims1, options, name1, name2, m_tid1, did1, did2); /* reclaim any VL memory, if necessary */ if(vl_data) { H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1); H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2); } /* end if */ /* calculate the next hyperslab offset */ for(i = rank1, carry = 1; i > 0 && carry; --i) { hs_offset[i - 1] += hs_size[i - 1]; if(hs_offset[i - 1] == dims1[i - 1]) hs_offset[i - 1] = 0; else carry = 0; } /* i */ } /* elmtno */ H5Sclose(sm_space); } /* hyperslab read */ } /*can_compare*/ /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ h5difftrace("compare attributes?\n"); /* free */ if(buf1 != NULL) { HDfree(buf1); buf1 = NULL; } /* end if */ if(buf2 != NULL) { HDfree(buf2); buf2 = NULL; } /* end if */ if(sm_buf1 != NULL) { HDfree(sm_buf1); sm_buf1 = NULL; } /* end if */ if(sm_buf2 != NULL) { HDfree(sm_buf2); sm_buf2 = NULL; } /* end if */ H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); } H5E_END_TRY; h5difftrace("diff_datasetid finish\n"); return nfound; error: options->err_stat=1; /* free */ if (buf1!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid1, sid1, H5P_DEFAULT, buf1); HDfree(buf1); buf1=NULL; } if (buf2!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid2, sid2, H5P_DEFAULT, buf2); HDfree(buf2); buf2=NULL; } if (sm_buf1!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1); HDfree(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2); HDfree(sm_buf2); sm_buf2=NULL; } /* disable error reporting */ H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); /* enable error reporting */ } H5E_END_TRY; h5difftrace("diff_datasetid errored\n"); return nfound; }
/*-------------------------------------------------------------------------
* Function: check_objects
*
* Purpose: locate all HDF5 objects in the file and compare with user
* supplied list
*
* Return: 0, ok, -1 no
*-------------------------------------------------------------------------
*/
static int check_objects(const char* fname, pack_opt_t *options) {
int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */
hid_t fid = -1;
hid_t did = -1;
hid_t sid = -1;
unsigned int i;
unsigned int uf;
trav_table_t *travt = NULL;
/* nothing to do */
if (options->op_tbl->nelems == 0)
HGOTO_DONE(0);
/*-------------------------------------------------------------------------
* open the file
*-------------------------------------------------------------------------
*/
if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, 0)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5tools_fopen failed <%s>: %s", fname, H5FOPENERROR);
/*-------------------------------------------------------------------------
* get the list of objects in the file
*-------------------------------------------------------------------------
*/
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
/* init table */
trav_table_init(&travt);
/* get the list of objects in the file */
if (h5trav_gettable(fid, travt) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed");
/*-------------------------------------------------------------------------
* compare with user supplied list
*-------------------------------------------------------------------------
*/
if (options->verbose)
printf("Opening file. Searching %d objects to modify ...\n", travt->nobjs);
for (i = 0; i < options->op_tbl->nelems; i++) {
char* name = options->op_tbl->objs[i].path;
if (options->verbose)
printf(" <%s>", name);
/* the input object names are present in the file and are valid */
if (h5trav_getindext(name, travt) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "%s Could not find <%s> in file <%s>. Exiting...\n",
(options->verbose ? "\n" : ""), name, fname);
if (options->verbose)
printf("...Found\n");
for (uf = 0; uf < options->op_tbl->objs[i].nfilters; uf++) {
if (options->op_tbl->objs[i].filter[uf].filtn < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid filter");
/* check for extra filter conditions */
switch (options->op_tbl->objs[i].filter[uf].filtn) {
/* chunk size must be smaller than pixels per block */
case H5Z_FILTER_SZIP:
{
int j;
hsize_t csize = 1;
unsigned ppb = options->op_tbl->objs[i].filter[uf].cd_values[0];
hsize_t dims[H5S_MAX_RANK];
int rank;
if (options->op_tbl->objs[i].chunk.rank > 0) {
rank = options->op_tbl->objs[i].chunk.rank;
for (j = 0; j < rank; j++)
csize *= options->op_tbl->objs[i].chunk.chunk_lengths[j];
}
else {
if ((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed");
if ((sid = H5Dget_space(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed");
if ((rank = H5Sget_simple_extent_ndims(sid)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed");
HDmemset(dims, 0, sizeof dims);
if (H5Sget_simple_extent_dims(sid, dims, NULL) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed");
for (j = 0; j < rank; j++)
csize *= dims[j];
if (H5Sclose(sid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed");
if (H5Dclose(did) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
}
if (csize < ppb) {
printf(" <warning: SZIP settings, chunk size is smaller than pixels per block>\n");
HGOTO_DONE(0);
}
}
break;
default:
break;
}
} /* for uf */
} /* for i */
done:
H5E_BEGIN_TRY {
H5Sclose(sid);
H5Dclose(did);
H5Fclose(fid);
} H5E_END_TRY;
if (travt)
trav_table_free(travt);
return ret_value;
}
/*-------------------------------------------------------------------------
* Function: H5O_layout_encode
*
* Purpose: Encodes a message.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* Wednesday, October 8, 1997
*
* Note:
* Quincey Koziol, 2004-5-21
* We write out version 3 messages by default now.
*
* Modifications:
* Robb Matzke, 1998-07-20
* Rearranged the message to add a version number at the beginning.
*
* Raymond Lu, 2002-2-26
* Added version number 2 case depends on if space has been allocated
* at the moment when layout header message is updated.
*
* Quincey Koziol, 2004-5-21
* Added version number 3 case to straighten out problems with contiguous
* layout's sizes (was encoding them as 4-byte values when they were
* really n-byte values (where n usually is 8)) and additionally clean up
* the information written out.
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const void *_mesg)
{
const H5O_layout_t *mesg = (const H5O_layout_t *) _mesg;
unsigned u;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5O_layout_encode)
/* check args */
HDassert(f);
HDassert(mesg);
HDassert(p);
/* Message version */
*p++ = (uint8_t)H5O_LAYOUT_VERSION_3;
/* Layout class */
*p++ = mesg->type;
/* Write out layout class specific information */
switch(mesg->type) {
case H5D_COMPACT:
/* Size of raw data */
UINT16ENCODE(p, mesg->storage.u.compact.size);
/* Raw data */
if(mesg->storage.u.compact.size > 0) {
if(mesg->storage.u.compact.buf)
HDmemcpy(p, mesg->storage.u.compact.buf, mesg->storage.u.compact.size);
else
HDmemset(p, 0, mesg->storage.u.compact.size);
p += mesg->storage.u.compact.size;
} /* end if */
break;
case H5D_CONTIGUOUS:
H5F_addr_encode(f, &p, mesg->storage.u.contig.addr);
H5F_ENCODE_LENGTH(f, p, mesg->storage.u.contig.size);
break;
case H5D_CHUNKED:
/* Number of dimensions */
HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS);
*p++ = (uint8_t)mesg->u.chunk.ndims;
/* B-tree address */
H5F_addr_encode(f, &p, mesg->storage.u.chunk.idx_addr);
/* Dimension sizes */
for(u = 0; u < mesg->u.chunk.ndims; u++)
UINT32ENCODE(p, mesg->u.chunk.dim[u]);
break;
default:
HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "Invalid layout class")
} /* end switch */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5O_layout_encode() */
hid_t
set_vfd(parameters *param)
{
hid_t my_fapl = -1;
vfdtype vfd;
vfd = param->vfd;
if ((my_fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1;
if (vfd == sec2) {
/* Unix read() and write() system calls */
if (H5Pset_fapl_sec2(my_fapl)<0) return -1;
} else if (vfd == stdio) {
/* Standard C fread() and fwrite() system calls */
if (H5Pset_fapl_stdio(my_fapl)<0) return -1;
} else if (vfd == core) {
/* In-core temporary file with 1MB increment */
if (H5Pset_fapl_core(my_fapl, (size_t)1024*1024, TRUE)<0) return -1;
} else if (vfd == split) {
/* Split meta data and raw data each using default driver */
if (H5Pset_fapl_split(my_fapl,
"-m.h5", H5P_DEFAULT,
"-r.h5", H5P_DEFAULT)<0)
return -1;
} else if (vfd == multi) {
/* Multi-file driver, general case of the split driver */
H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
hid_t memb_fapl[H5FD_MEM_NTYPES];
const char *memb_name[H5FD_MEM_NTYPES];
char sv[H5FD_MEM_NTYPES][1024];
haddr_t memb_addr[H5FD_MEM_NTYPES];
H5FD_mem_t mt;
HDmemset(memb_map, 0, sizeof memb_map);
HDmemset(memb_fapl, 0, sizeof memb_fapl);
HDmemset(memb_name, 0, sizeof memb_name);
HDmemset(memb_addr, 0, sizeof memb_addr);
HDassert(HDstrlen(multi_letters)==H5FD_MEM_NTYPES);
for (mt=H5FD_MEM_DEFAULT; mt<H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t,mt)) {
memb_fapl[mt] = H5P_DEFAULT;
sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
memb_name[mt] = sv[mt];
memb_addr[mt] = (haddr_t)MAX(mt - 1,0) * (HADDR_MAX / 10);
}
if (H5Pset_fapl_multi(my_fapl, memb_map, memb_fapl, memb_name,
memb_addr, FALSE)<0) {
return -1;
}
} else if (vfd == family) {
hsize_t fam_size = 1*1024*1024; /*100 MB*/
/* Family of files, each 1MB and using the default driver */
/* if ((val=HDstrtok(NULL, " \t\n\r")))
fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024); */
if (H5Pset_fapl_family(my_fapl, fam_size, H5P_DEFAULT)<0)
return -1;
} else if (vfd == direct) {
#ifdef H5_HAVE_DIRECT
/* Linux direct read() and write() system calls. Set memory boundary, file block size,
* and copy buffer size to the default values. */
if (H5Pset_fapl_direct(my_fapl, 1024, 4096, 8*4096)<0) return -1;
#endif
} else {
/* Unknown driver */
return -1;
}
return my_fapl;
}
static intn test_szip_SDSfl64bit()
{
/************************* Variable declaration **************************/
int32 sd_id, sds_id;
intn status;
int32 dim_sizes[2], array_rank, num_type, attributes;
char name[H4_MAX_NC_NAME];
comp_info c_info;
int32 start[2], edges[2];
float64 fill_value = 0; /* Fill value */
int i,j;
int num_errs = 0; /* number of errors so far */
float64 out_data[LENGTH][WIDTH];
float64 in_data[LENGTH][WIDTH]={
100.0,100.0,200.0,200.0,300.0,400.0,
100.0,100.0,200.0,200.0,300.0,400.0,
100.0,100.0,200.0,200.0,300.0,400.0,
300.0,300.0, 0.0,400.0,300.0,400.0,
300.0,300.0, 0.0,400.0,300.0,400.0,
300.0,300.0, 0.0,400.0,300.0,400.0,
0.0, 0.0,600.0,600.0,300.0,400.0,
500.0,500.0,600.0,600.0,300.0,400.0,
0.0, 0.0,600.0,600.0,300.0,400.0};
/********************* End of variable declaration ***********************/
/* Create the file and initialize SD interface */
sd_id = SDstart (FILE_NAMEfl64, DFACC_CREATE);
CHECK(sd_id, FAIL, "SDstart");
/* Create the SDS */
dim_sizes[0] = LENGTH;
dim_sizes[1] = WIDTH;
sds_id = SDcreate (sd_id, SDS_NAME, DFNT_FLOAT64, RANK, dim_sizes);
CHECK(sds_id, FAIL, "SDcreate:Failed to create a data set for szip compression testing");
/* Define the location, pattern, and size of the data set */
for (i = 0; i < RANK; i++) {
start[i] = 0;
edges[i] = dim_sizes[i];
}
/* Fill the SDS array with the fill value */
status = SDsetfillvalue (sds_id, (VOIDP)&fill_value);
CHECK(status, FAIL, "SDsetfillvalue");
/* Initialization for SZIP */
c_info.szip.pixels_per_block = 2;
c_info.szip.options_mask = SZ_EC_OPTION_MASK;
c_info.szip.options_mask |= SZ_RAW_OPTION_MASK;
c_info.szip.bits_per_pixel = 0;
c_info.szip.pixels = 0;
c_info.szip.pixels_per_scanline = 0;
/* Set the compression */
status = SDsetcompress (sds_id, COMP_CODE_SZIP, &c_info);
CHECK(status, FAIL, "SDsetcompress");
/* Write data to the SDS */
status = SDwritedata(sds_id, start, NULL, edges, (VOIDP)in_data);
CHECK(status, FAIL, "SDwritedata");
/* Terminate access to the data set */
status = SDendaccess (sds_id);
CHECK(status, FAIL, "SDendaccess");
/* Terminate access to the SD interface and close the file to
flush the compressed info to the file */
status = SDend (sd_id);
CHECK(status, FAIL, "SDend");
/*
* Verify the compressed data
*/
/* Reopen the file and select the first SDS */
sd_id = SDstart (FILE_NAMEfl64, DFACC_READ);
CHECK(sd_id, FAIL, "SDstart");
sds_id = SDselect (sd_id, 0);
CHECK(sds_id, FAIL, "SDselect:Failed to select a data set for szip compression testing");
/* Retrieve information of the data set */
status = SDgetinfo(sds_id, name, &array_rank, dim_sizes, &num_type, &attributes);
CHECK(status, FAIL, "SDgetinfo");
/* Wipe out the output buffer */
HDmemset(&out_data, 0, sizeof(out_data));
/* Read the data set */
start[0] = 0;
start[1] = 0;
edges[0] = LENGTH;
edges[1] = WIDTH;
status = SDreaddata (sds_id, start, NULL, edges, (VOIDP)out_data);
CHECK(status, FAIL, "SDreaddata");
/* Compare read data against input data */
for (j=0; j<LENGTH; j++)
{
for (i=0; i<WIDTH; i++)
if (out_data[j][i] != in_data[j][i])
{
fprintf(stderr,"Bogus val in loc [%d][%d] in compressed dset, want %ld got %ld\n", j, i, (long)in_data[j][i], (long)out_data[j][i]);
num_errs++;
}
}
/* Terminate access to the data set */
status = SDendaccess (sds_id);
CHECK(status, FAIL, "SDendaccess");
/* Terminate access to the SD interface and close the file */
status = SDend (sd_id);
CHECK(status, FAIL, "SDend");
/* Return the number of errors that's been kept track of so far */
return num_errs;
} /* test_szip_SDSfl64bit */
/*-------------------------------------------------------------------------
* Function: H5Z_filter_deflate
*
* Purpose: Implement an I/O filter around the 'deflate' algorithm in
* libz
*
* Return: Success: Size of buffer filtered
* Failure: 0
*
* Programmer: Robb Matzke
* Thursday, April 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static size_t
H5Z_filter_deflate (unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void *outbuf = NULL; /* Pointer to new buffer */
int status; /* Status from zlib operation */
size_t ret_value; /* Return value */
FUNC_ENTER_NOAPI(0)
/* Sanity check */
HDassert(*buf_size > 0);
HDassert(buf);
HDassert(*buf);
/* Check arguments */
if (cd_nelmts!=1 || cd_values[0]>9)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level")
if (flags & H5Z_FLAG_REVERSE) {
/* Input; uncompress */
z_stream z_strm; /* zlib parameters */
size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */
/* Allocate space for the compressed buffer */
if (NULL==(outbuf = H5MM_malloc(nalloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression")
/* Set the uncompression parameters */
HDmemset(&z_strm, 0, sizeof(z_strm));
z_strm.next_in = (Bytef *)*buf;
H5_ASSIGN_OVERFLOW(z_strm.avail_in,nbytes,size_t,unsigned);
z_strm.next_out = (Bytef *)outbuf;
H5_ASSIGN_OVERFLOW(z_strm.avail_out,nalloc,size_t,unsigned);
/* Initialize the uncompression routines */
if (Z_OK!=inflateInit(&z_strm))
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed")
/* Loop to uncompress the buffer */
do {
/* Uncompress some data */
status = inflate(&z_strm, Z_SYNC_FLUSH);
/* Check if we are done uncompressing data */
if (Z_STREAM_END==status)
break; /*done*/
/* Check for error */
if (Z_OK!=status) {
(void)inflateEnd(&z_strm);
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed")
}
else {
/* If we're not done and just ran out of buffer space, get more */
if(0 == z_strm.avail_out) {
void *new_outbuf; /* Pointer to new output buffer */
/* Allocate a buffer twice as big */
nalloc *= 2;
if(NULL == (new_outbuf = H5MM_realloc(outbuf, nalloc))) {
(void)inflateEnd(&z_strm);
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression")
} /* end if */
outbuf = new_outbuf;
/* Update pointers to buffer for next set of uncompressed data */
z_strm.next_out = (unsigned char*)outbuf + z_strm.total_out;
z_strm.avail_out = (uInt)(nalloc - z_strm.total_out);
} /* end if */
static intn test_szip_chunk_3d()
{
/************************* Variable declaration **************************/
int32 sd_id, sds_id0, sds_id, sds_index;
intn status;
int32 dim_sizes[3];
HDF_CHUNK_DEF c_def; /* Chunking definitions */
HDF_CHUNK_DEF c_def_out; /* Chunking definitions */
int32 c_flags, c_flags_out;
int32 start[3], edges[3];
int16 fill_value = 0; /* Fill value */
comp_coder_t comp_type; /* to retrieve compression type into */
comp_info cinfo; /* compression information structure */
int num_errs = 0; /* number of errors so far */
int i,j,k;
for (i = 0; i < SDS_DIM0; i++) {
for (j = 0; j < SDS_DIM1; j++) {
for (k = 0; k < SDS_DIM2; k++) {
out_data[i][j][k] = i*100+j*10+k;
}}}
/* Initialize chunk lengths. */
c_def.comp.chunk_lengths[0] = CHK_DIM0;
c_def.comp.chunk_lengths[1] = CHK_DIM1;
c_def.comp.chunk_lengths[2] = CHK_DIM2;
/* Create the file and initialize SD interface. */
sd_id = SDstart (FILE_NAME_3D, DFACC_CREATE);
CHECK(sd_id, FAIL, "SDstart");
/* Create SDS_DIM0xSDS_DIM1 SDS. */
dim_sizes[0] = SDS_DIM0;
dim_sizes[1] = SDS_DIM1;
dim_sizes[2] = SDS_DIM2;
sds_id = SDcreate (sd_id, SDS_NAME_CH3D, DFNT_INT16, RANK_CH3, dim_sizes);
CHECK(sds_id, FAIL, "SDcreate:Failed to create a data set for chunking/szip compression testing");
/* Create a similar SDS and will make it chunked, but will not
write data to it */
sds_id0 = SDcreate (sd_id, SDS_NAME_CH0, DFNT_INT16, RANK_CH3, dim_sizes);
CHECK(sds_id0, FAIL, "SDcreate:Failed to create a data set for chunking/szip compression testing");
/* Fill the SDS array with the fill value. */
status = SDsetfillvalue (sds_id, (VOIDP)&fill_value);
CHECK(status, FAIL, "SDsetfillvalue");
/* Set parameters for Chunking/SZIP */
c_def.comp.comp_type = COMP_CODE_SZIP;
c_def.comp.cinfo.szip.pixels_per_block = 2;
c_def.comp.cinfo.szip.options_mask = SZ_EC_OPTION_MASK;
c_def.comp.cinfo.szip.options_mask |= SZ_MSB_OPTION_MASK;
c_def.comp.cinfo.szip.bits_per_pixel = 2;
c_def.comp.cinfo.szip.pixels = 16;
c_def.comp.cinfo.szip.pixels_per_scanline = 2;
c_flags = HDF_CHUNK | HDF_COMP;
status = SDsetchunk (sds_id0, c_def, c_flags);
status = SDsetchunk (sds_id, c_def, c_flags);
CHECK(status, FAIL, "SDsetchunk");
start[0] = 0;
start[1] = 0;
start[2] = 0;
edges[0] = SDS_DIM0;
edges[1] = SDS_DIM1;
edges[2] = SDS_DIM2;
status = SDwritedata (sds_id, start, NULL, edges, (VOIDP) out_data);
CHECK(status, FAIL, "SDwritedata");
HDmemset(&c_def_out, 0, sizeof(HDF_CHUNK_DEF));
c_flags_out = 0;
status = SDgetchunkinfo(sds_id0, &c_def_out, &c_flags_out);
CHECK(status, FAIL, "SDgetchunkinfo");
VERIFY(c_flags_out, c_flags, "SDgetchunkinfo");
VERIFY(c_def_out.comp.comp_type, COMP_CODE_SZIP, "SDgetchunkinfo");
HDmemset(&c_def_out, 0, sizeof(HDF_CHUNK_DEF));
c_flags_out = 0;
status = SDgetchunkinfo(sds_id, &c_def_out, &c_flags_out);
CHECK(status, FAIL, "SDgetchunkinfo");
VERIFY(c_flags_out, c_flags, "SDgetchunkinfo");
VERIFY(c_def_out.comp.comp_type, COMP_CODE_SZIP, "SDgetchunkinfo");
/* Terminate access to the data sets. */
status = SDendaccess (sds_id0);
CHECK(status, FAIL, "SDendaccess");
status = SDendaccess (sds_id);
CHECK(status, FAIL, "SDendaccess");
/* Terminate access to the SD interface and close the file. */
status = SDend (sd_id);
CHECK(status, FAIL, "SDend");
/*
* Verify the compressed data
*/
/* Reopen the file and access the first data set. */
sd_id = SDstart (FILE_NAME_3D, DFACC_READ);
sds_index = 0;
sds_id = SDselect (sd_id, sds_index);
CHECK(sds_id, FAIL, "SDselect:Failed to select a data set for chunking/szip compression testing");
/* Retrieve compression information about the dataset */
comp_type = COMP_CODE_INVALID; /* reset variables before retrieving info */
HDmemset(&cinfo, 0, sizeof(cinfo)) ;
status = SDgetcompinfo(sds_id, &comp_type, &cinfo);
CHECK(status, FAIL, "SDgetcompinfo");
VERIFY(comp_type, COMP_CODE_SZIP, "SDgetcompinfo");
/* Retrieve compression method alone from the dataset */
comp_type = COMP_CODE_INVALID; /* reset variables before retrieving info */
status = SDgetcomptype(sds_id, &comp_type);
CHECK(status, FAIL, "SDgetcomptype");
VERIFY(comp_type, COMP_CODE_SZIP, "SDgetcomptype");
start[0] = 0;
start[1] = 0;
start[2] = 0;
edges[0] = SDS_DIM0;
edges[1] = SDS_DIM1;
edges[2] = SDS_DIM2;
status = SDreaddata (sds_id, start, NULL, edges, (VOIDP)all_data);
CHECK(status, FAIL, "SDreaddata");
for (i = 0; i < SDS_DIM0; i++) {
for (j = 0; j < SDS_DIM1; j++) {
for (k = 0; k < SDS_DIM2; k++) {
if (out_data[i][j][k] != all_data[i][j][k])
{
fprintf(stderr,"Bogus val in loc [%d][%d][%d] want %ld got %ld\n", i, j,k, out_data[i][j][k], all_data[i][j][k]);
num_errs++;
}
}
}
}
/* Terminate access to the data set. */
status = SDendaccess (sds_id);
CHECK(status, FAIL, "SDendaccess");
/* Terminate access to the SD interface and close the file. */
status = SDend (sd_id);
CHECK(status, FAIL, "SDend");
/* Return the number of errors that's been kept track of so far */
return num_errs;
} /* test_szip_chunk_3D */
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose:
*
* Return: Success:
*
* Failure:
*
* Programmer: Robb Matzke
* Thursday, March 12, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
main (void)
{
static hsize_t size[2] = {REQUEST_SIZE_X, REQUEST_SIZE_Y};
static unsigned nread = NREAD_REQUESTS, nwrite = NWRITE_REQUESTS;
unsigned char *the_data = NULL;
hid_t file, dset, file_space = -1;
herr_t status;
#ifdef H5_HAVE_GETRUSAGE
struct rusage r_start, r_stop;
#else
struct timeval r_start, r_stop;
#endif
struct timeval t_start, t_stop;
int fd;
unsigned u;
hssize_t n;
off_t offset;
hsize_t start[2];
hsize_t count[2];
#ifdef H5_HAVE_SYS_TIMEB
struct _timeb *tbstart = malloc(sizeof(struct _timeb));
struct _timeb *tbstop = malloc(sizeof(struct _timeb));
#endif
/*
* The extra cast in the following statement is a bug workaround for the
* Win32 version 5.0 compiler.
* 1998-11-06 ptl
*/
printf ("I/O request size is %1.1fMB\n",
(double)(hssize_t)(size[0]*size[1])/1024.0F*1024);
/* Open the files */
file = H5Fcreate (HDF5_FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
assert (file>=0);
fd = HDopen (RAW_FILE_NAME, O_RDWR|O_CREAT|O_TRUNC, 0666);
assert (fd>=0);
/* Create the dataset */
file_space = H5Screate_simple (2, size, size);
assert(file_space >= 0);
dset = H5Dcreate2(file, "dset", H5T_NATIVE_UCHAR, file_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
assert(dset >= 0);
the_data = (unsigned char *)malloc((size_t)(size[0] * size[1]));
/* initial fill for lazy malloc */
HDmemset(the_data, 0xAA, (size_t)(size[0] * size[1]));
/* Fill raw */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "fill raw");
for(u = 0; u < nwrite; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
HDmemset(the_data, 0xAA, (size_t)(size[0]*size[1]));
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("fill raw",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Fill hdf5 */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "fill hdf5");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("fill hdf5",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Write the raw dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "out raw");
for(u = 0; u < nwrite; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
offset = HDlseek (fd, (off_t)0, SEEK_SET);
assert (0==offset);
n = HDwrite (fd, the_data, (size_t)(size[0]*size[1]));
assert (n>=0 && (size_t)n==size[0]*size[1]);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("out raw",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Write the hdf5 dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "out hdf5");
for(u = 0; u < nwrite; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dwrite (dset, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("out hdf5",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Read the raw dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "in raw");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
offset = HDlseek (fd, (off_t)0, SEEK_SET);
assert (0==offset);
n = HDread (fd, the_data, (size_t)(size[0]*size[1]));
assert (n>=0 && (size_t)n==size[0]*size[1]);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("in raw",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Read the hdf5 dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "in hdf5");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("in hdf5",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Read hyperslab */
assert (size[0]>20 && size[1]>20);
start[0] = start[1] = 10;
count[0] = count[1] = size[0]-20;
status = H5Sselect_hyperslab (file_space, H5S_SELECT_SET, start, NULL, count, NULL);
assert (status>=0);
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "in hdf5 partial");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc('\n', stderr);
print_stats("in hdf5 partial",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*count[0]*count[1]));
/* Close everything */
HDclose(fd);
H5Dclose(dset);
H5Sclose(file_space);
H5Fclose(file);
free(the_data);
return 0;
}
/*--------------------------------------------------------------------------
NAME
H5O_attr_encode
PURPOSE
Encode a simple attribute message
USAGE
herr_t H5O_attr_encode(f, raw_size, p, mesg)
H5F_t *f; IN: pointer to the HDF5 file struct
const uint8 *p; IN: the raw information buffer
const void *mesg; IN: Pointer to the simple datatype struct
RETURNS
Non-negative on success/Negative on failure
DESCRIPTION
This function encodes the native memory form of the attribute
message in the "raw" disk form.
*
* Modifications:
* Robb Matzke, 17 Jul 1998
* Added padding for alignment.
*
* Robb Matzke, 20 Jul 1998
* Added a version number at the beginning.
*
* Raymond Lu, 8 April 2004
* For data space, changed the operation on H5S_simple_t to
* H5S_extent_t
*
--------------------------------------------------------------------------*/
static herr_t
H5O_attr_encode(H5F_t *f, uint8_t *p, const void *mesg)
{
const H5A_t *attr = (const H5A_t *) mesg;
size_t name_len; /* Attribute name length */
unsigned version; /* Attribute version */
hbool_t type_shared; /* Flag to indicate that a shared datatype is used for this attribute */
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5O_attr_encode);
/* check args */
assert(f);
assert(p);
assert(attr);
/* Check whether datatype is shared */
if(H5T_committed(attr->dt))
type_shared = TRUE;
else
type_shared = FALSE;
/* Check which version to write out */
if(type_shared)
version = H5O_ATTR_VERSION_NEW; /* Write out new version if shared datatype */
else
version = H5O_ATTR_VERSION;
/* Encode Version */
*p++ = version;
/* Set attribute flags if version >1 */
if(version>H5O_ATTR_VERSION)
*p++ = (type_shared ? H5O_ATTR_FLAG_TYPE_SHARED : 0 ); /* Set flags for attribute */
else
*p++ = 0; /* Reserved, for version <2 */
/*
* Encode the lengths of the various parts of the attribute message. The
* encoded lengths are exact but we pad each part except the data to be a
* multiple of eight bytes (in the first version).
*/
name_len = HDstrlen(attr->name)+1;
UINT16ENCODE(p, name_len);
UINT16ENCODE(p, attr->dt_size);
UINT16ENCODE(p, attr->ds_size);
/*
* Write the name including null terminator padded to the correct number
* of bytes.
*/
HDmemcpy(p, attr->name, name_len);
HDmemset(p+name_len, 0, H5O_ALIGN(name_len)-name_len);
if(version < H5O_ATTR_VERSION_NEW)
p += H5O_ALIGN(name_len);
else
p += name_len;
/* encode the attribute datatype */
if(type_shared) {
H5O_shared_t sh_mesg;
/* Reset shared message information */
HDmemset(&sh_mesg,0,sizeof(H5O_shared_t));
/* Get shared message information from datatype */
if ((H5O_MSG_DTYPE->get_share)(f, attr->dt, &sh_mesg/*out*/)<0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTENCODE, FAIL, "can't encode shared attribute datatype");
/* Encode shared message information for datatype */
if((H5O_MSG_SHARED->encode)(f,p,&sh_mesg)<0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTENCODE, FAIL, "can't encode shared attribute datatype");
} /* end if */
else {
/* Encode datatype information */
if((H5O_MSG_DTYPE->encode)(f,p,attr->dt)<0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTENCODE, FAIL, "can't encode attribute datatype");
} /* end else */
if(version < H5O_ATTR_VERSION_NEW) {
HDmemset(p+attr->dt_size, 0, H5O_ALIGN(attr->dt_size)-attr->dt_size);
p += H5O_ALIGN(attr->dt_size);
} /* end if */
else
p += attr->dt_size;
/* encode the attribute dataspace */
if((H5O_MSG_SDSPACE->encode)(f,p,&(attr->ds->extent))<0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTENCODE, FAIL, "can't encode attribute dataspace");
if(version < H5O_ATTR_VERSION_NEW) {
HDmemset(p+attr->ds_size, 0, H5O_ALIGN(attr->ds_size)-attr->ds_size);
p += H5O_ALIGN(attr->ds_size);
} /* end if */
else
p += attr->ds_size;
/* Store attribute data */
if(attr->data)
HDmemcpy(p,attr->data,attr->data_size);
else
HDmemset(p,0,attr->data_size);
done:
FUNC_LEAVE_NOAPI(ret_value);
}
static void
gent_ub(const char * filename, size_t ub_size, size_t ub_fill)
{
hid_t fid, group, attr, dataset, space;
hid_t create_plist;
hsize_t dims[2];
int data[2][2], dset1[10][10], dset2[20];
char buf[BUF_SIZE];
int i, j;
size_t u;
float dset2_1[10], dset2_2[3][5];
int fd;
char *bp;
if(ub_size > 0)
{
create_plist = H5Pcreate(H5P_FILE_CREATE);
H5Pset_userblock(create_plist, (hsize_t)ub_size);
fid = H5Fcreate(filename, H5F_ACC_TRUNC, create_plist, H5P_DEFAULT);
}
else
{
fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
}
/* create groups */
group = H5Gcreate2(fid, "/g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Gclose(group);
group = H5Gcreate2(fid, "/g2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Gclose(group);
group = H5Gcreate2(fid, "/g1/g1.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Gclose(group);
group = H5Gcreate2(fid, "/g1/g1.2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Gclose(group);
group = H5Gcreate2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Gclose(group);
/* root attributes */
group = H5Gopen2(fid, "/", H5P_DEFAULT);
dims[0] = 10;
space = H5Screate_simple(1, dims, NULL);
attr = H5Acreate2(group, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT);
sprintf(buf, "abcdefghi");
H5Awrite(attr, H5T_NATIVE_SCHAR, buf);
H5Sclose(space);
H5Aclose(attr);
dims[0] = 2; dims[1] = 2;
space = H5Screate_simple(2, dims, NULL);
attr = H5Acreate2(group, "attr2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT);
data[0][0] = 0; data[0][1] = 1; data[1][0] = 2; data[1][1] = 3;
H5Awrite(attr, H5T_NATIVE_INT, data);
H5Sclose(space);
H5Aclose(attr);
H5Gclose(group);
group = H5Gopen2(fid, "/g1/g1.1", H5P_DEFAULT);
/* dset1.1.1 */
dims[0] = 10; dims[1] = 10;
space = H5Screate_simple(2, dims, NULL);
dataset = H5Dcreate2(group, "dset1.1.1", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
for (i = 0; i < 10; i++)
for (j = 0; j < 10; j++)
dset1[i][j] = j*i;
H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset1);
H5Sclose(space);
/* attributes of dset1.1.1 */
dims[0] = 27;
space = H5Screate_simple(1, dims, NULL);
attr = H5Acreate2(dataset, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT);
sprintf(buf, "1st attribute of dset1.1.1");
H5Awrite(attr, H5T_NATIVE_SCHAR, buf);
H5Sclose(space);
H5Aclose(attr);
dims[0] = 27;
space = H5Screate_simple(1, dims, NULL);
attr = H5Acreate2(dataset, "attr2", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT);
sprintf(buf, "2nd attribute of dset1.1.1");
H5Awrite(attr, H5T_NATIVE_SCHAR, buf);
H5Sclose(space);
H5Aclose(attr);
H5Dclose(dataset);
/* dset1.1.2 */
dims[0] = 20;
space = H5Screate_simple(1, dims, NULL);
dataset = H5Dcreate2(group, "dset1.1.2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
for (i = 0; i < 20; i++)
dset2[i] = i;
H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2);
H5Sclose(space);
H5Dclose(dataset);
H5Gclose(group);
/* external link */
H5Lcreate_external("somefile", "somepath", fid, "/g1/g1.2/extlink", H5P_DEFAULT, H5P_DEFAULT);
/* soft link */
group = H5Gopen2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT);
H5Lcreate_soft("somevalue", group, "slink", H5P_DEFAULT, H5P_DEFAULT);
H5Gclose(group);
group = H5Gopen2(fid, "/g2", H5P_DEFAULT);
/* dset2.1 */
dims[0] = 10;
space = H5Screate_simple(1, dims, NULL);
dataset = H5Dcreate2(group, "dset2.1", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
for (i = 0; i < 10; i++)
dset2_1[i] = (float)((float)i * 0.1F + 1.0F);
H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_1);
H5Sclose(space);
H5Dclose(dataset);
/* dset2.2 */
dims[0] = 3; dims[1] = 5;
space = H5Screate_simple(2, dims, NULL);
dataset = H5Dcreate2(group, "dset2.2", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
for (i = 0; i < 3; i++)
for (j = 0; j < 5; j++)
dset2_2[i][j] = (float)(((float)i + 1.0F) * (float)j * 0.1F);
H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_2);
H5Sclose(space);
H5Dclose(dataset);
H5Gclose(group);
/* user-defined link */
H5Lregister(UD_link_class);
H5Lcreate_ud(fid, "/g2/udlink", (H5L_type_t)MY_LINKCLASS, NULL, (size_t)0, H5P_DEFAULT, H5P_DEFAULT);
H5Fclose(fid);
/* If a user block is being used, write to it here */
if(ub_size > 0)
{
ssize_t nbytes;
HDassert(ub_size <= BUF_SIZE);
fd = HDopen(filename, O_RDWR);
HDassert(fd >= 0);
/* fill buf with pattern */
HDmemset(buf, '\0', ub_size);
bp = buf;
for (u = 0; u < ub_fill; u++)
*bp++ = pattern[u % 10];
nbytes = HDwrite(fd, buf, ub_size);
HDassert(nbytes >= 0);
HDclose(fd);
}
}
int
main (int argc, const char *argv[])
{
hid_t fid_src = -1;
hid_t fid_dst = -1;
unsigned flag = 0;
unsigned verbose = 0;
unsigned parents = 0;
hid_t ocpl_id = (-1); /* Object copy property list */
hid_t lcpl_id = (-1); /* Link creation property list */
int opt;
int li_ret;
h5tool_link_info_t linkinfo;
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
/* initialize h5tools lib */
h5tools_init();
/* init linkinfo struct */
HDmemset(&linkinfo, 0, sizeof(h5tool_link_info_t));
/* Check for no command line parameters */
if(argc == 1)
{
usage();
leave(EXIT_FAILURE);
} /* end if */
/* parse command line options */
while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF)
{
switch ((char)opt)
{
case 'd':
oname_dst = HDstrdup(opt_arg);
break;
case 'f':
/* validate flag */
if (parse_flag(opt_arg,&flag)<0)
{
usage();
leave(EXIT_FAILURE);
}
str_flag = HDstrdup(opt_arg);
break;
case 'h':
usage();
leave(EXIT_SUCCESS);
break;
case 'i':
fname_src = HDstrdup(opt_arg);
break;
case 'o':
fname_dst = HDstrdup(opt_arg);
break;
case 'p':
parents = 1;
break;
case 's':
oname_src = HDstrdup(opt_arg);
break;
case 'V':
print_version(h5tools_getprogname());
leave(EXIT_SUCCESS);
break;
case 'v':
verbose = 1;
break;
default:
usage();
leave(EXIT_FAILURE);
}
} /* end of while */
/*-------------------------------------------------------------------------
* check for missing file/object names
*-------------------------------------------------------------------------*/
if (fname_src==NULL)
{
error_msg("Input file name missing\n");
usage();
leave(EXIT_FAILURE);
}
if (fname_dst==NULL)
{
error_msg("Output file name missing\n");
usage();
leave(EXIT_FAILURE);
}
if (oname_src==NULL)
{
error_msg("Source object name missing\n");
usage();
leave(EXIT_FAILURE);
}
if (oname_dst==NULL)
{
error_msg("Destination object name missing\n");
usage();
leave(EXIT_FAILURE);
}
/*-------------------------------------------------------------------------
* open output file
*-------------------------------------------------------------------------*/
/* Attempt to open an existing HDF5 file first. Need to open the dst file
before the src file just in case that the dst and src are the same file
*/
fid_dst = h5tools_fopen(fname_dst, H5F_ACC_RDWR, H5P_DEFAULT, NULL, NULL, 0);
/*-------------------------------------------------------------------------
* open input file
*-------------------------------------------------------------------------*/
fid_src = h5tools_fopen(fname_src, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, 0);
/*-------------------------------------------------------------------------
* test for error in opening input file
*-------------------------------------------------------------------------*/
if (fid_src==-1)
{
error_msg("Could not open input file <%s>...Exiting\n", fname_src);
leave(EXIT_FAILURE);
}
/*-------------------------------------------------------------------------
* create an output file when failed to open it
*-------------------------------------------------------------------------*/
/* If we couldn't open an existing file, try creating file */
/* (use "EXCL" instead of "TRUNC", so we don't blow away existing non-HDF5 file) */
if(fid_dst < 0)
fid_dst = H5Fcreate(fname_dst, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT);
/*-------------------------------------------------------------------------
* test for error in opening output file
*-------------------------------------------------------------------------*/
if (fid_dst==-1)
{
error_msg("Could not open output file <%s>...Exiting\n", fname_dst);
leave(EXIT_FAILURE);
}
/*-------------------------------------------------------------------------
* print some info
*-------------------------------------------------------------------------*/
if (verbose)
{
printf("Copying file <%s> and object <%s> to file <%s> and object <%s>\n",
fname_src, oname_src, fname_dst, oname_dst);
if (flag) {
HDassert(str_flag);
printf("Using %s flag\n", str_flag);
}
}
/*-------------------------------------------------------------------------
* create property lists for copy
*-------------------------------------------------------------------------*/
/* create property to pass copy options */
if ( (ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0)
goto error;
/* set options for object copy */
if (flag)
{
if ( H5Pset_copy_object(ocpl_id, flag) < 0)
goto error;
}
/* Create link creation property list */
if((lcpl_id = H5Pcreate(H5P_LINK_CREATE)) < 0) {
error_msg("Could not create link creation property list\n");
goto error;
} /* end if */
/* Check for creating intermediate groups */
if(parents) {
/* Set the intermediate group creation property */
if(H5Pset_create_intermediate_group(lcpl_id, 1) < 0) {
error_msg("Could not set property for creating parent groups\n");
goto error;
} /* end if */
/* Display some output if requested */
if(verbose)
printf("%s: Creating parent groups\n", h5tools_getprogname());
} /* end if */
else /* error, if parent groups doesn't already exist in destination file */
{
size_t i, len;
len = HDstrlen(oname_dst);
/* check if all the parents groups exist. skip root group */
for (i = 1; i < len; i++)
{
if ('/'==oname_dst[i])
{
char *str_ptr;
str_ptr = (char *)HDcalloc(i + 1, sizeof(char));
HDstrncpy(str_ptr, oname_dst, i);
str_ptr[i]='\0';
if (H5Lexists(fid_dst, str_ptr, H5P_DEFAULT) <= 0)
{
error_msg("group <%s> doesn't exist. Use -p to create parent groups.\n", str_ptr);
HDfree(str_ptr);
goto error;
}
HDfree(str_ptr);
}
}
}
/*-------------------------------------------------------------------------
* do the copy
*-------------------------------------------------------------------------*/
if(verbose)
linkinfo.opt.msg_mode = 1;
li_ret = H5tools_get_symlink_info(fid_src, oname_src, &linkinfo, 1);
if (li_ret == 0) /* dangling link */
{
if(H5Lcopy(fid_src, oname_src,
fid_dst, oname_dst,
H5P_DEFAULT, H5P_DEFAULT) < 0)
goto error;
}
else /* valid link */
{
if (H5Ocopy(fid_src, /* Source file or group identifier */
oname_src, /* Name of the source object to be copied */
fid_dst, /* Destination file or group identifier */
oname_dst, /* Name of the destination object */
ocpl_id, /* Object copy property list */
lcpl_id)<0) /* Link creation property list */
goto error;
}
/* free link info path */
if (linkinfo.trg_path)
HDfree(linkinfo.trg_path);
/* close propertis */
if(H5Pclose(ocpl_id)<0)
goto error;
if(H5Pclose(lcpl_id)<0)
goto error;
/* close files */
if (H5Fclose(fid_src)<0)
goto error;
if (H5Fclose(fid_dst)<0)
goto error;
leave(EXIT_SUCCESS);
error:
printf("Error in copy...Exiting\n");
/* free link info path */
if (linkinfo.trg_path)
HDfree(linkinfo.trg_path);
H5E_BEGIN_TRY {
H5Pclose(ocpl_id);
H5Pclose(lcpl_id);
H5Fclose(fid_src);
H5Fclose(fid_dst);
} H5E_END_TRY;
leave(EXIT_FAILURE);
}
/*-------------------------------------------------------------------------
* Function: doprint()
*
* Purpose: Prepare to print the dataset's appended data.
* Call the tools library routine h5tools_dump_dset() to do the printing.
* (This routine is mostly copied from dump_dataset_values() in tools/h5ls/h5ls.c
* and modified accordingly).
*
* Return: 0 on success; negative on failure
*
* Programmer: Vailin Choi; August 2010
*
*-------------------------------------------------------------------------
*/
static herr_t
doprint(hid_t did, hsize_t *start, hsize_t *block, int rank)
{
h5tools_context_t ctx; /* print context */
h5tool_format_t info; /* Format info for the tools library */
static char fmt_double[16], fmt_float[16]; /* Format info */
struct subset_t subset; /* Subsetting info */
hsize_t ss_start[H5S_MAX_RANK]; /* Info for hyperslab */
hsize_t ss_stride[H5S_MAX_RANK]; /* Info for hyperslab */
hsize_t ss_block[H5S_MAX_RANK]; /* Info for hyperslab */
hsize_t ss_count[H5S_MAX_RANK]; /* Info for hyperslab */
int i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
/* Subsetting information for the tools library printing routines */
subset.start.data = ss_start;
subset.stride.data = ss_stride;
subset.block.data = ss_block;
subset.count.data = ss_count;
/* Initialize subsetting information */
for(i = 0; i < rank; i++) {
subset.stride.data[i] = 1;
subset.count.data[i] = 1;
subset.start.data[i] = start[i];
subset.block.data[i] = block[i];
}
HDmemset(&ctx, 0, sizeof(ctx));
/* Set to all default values and then override */
HDmemset(&info, 0, sizeof info);
if(g_simple_output) {
info.idx_fmt = "";
info.line_ncols = 65535; /*something big*/
info.line_per_line = 1;
info.line_multi_new = 0;
info.line_pre = " ";
info.line_cont = " ";
info.arr_pre = "";
info.arr_suf = "";
info.arr_sep = " ";
info.cmpd_pre = "";
info.cmpd_suf = "";
info.cmpd_sep = " ";
/* The "fields" selected by the user */
info.cmpd_listv = (const struct H5LD_memb_t **)g_listv;
if(g_label) info.cmpd_name = "%s=";
info.elmt_suf1 = " ";
info.str_locale = ESCAPE_HTML;
} else {
info.idx_fmt = "(%s)";
if(!g_display_width) {
info.line_ncols = 65535;
info.line_per_line = 1;
}
else
info.line_ncols = (unsigned)g_display_width;
info.line_multi_new = 1;
/* The "fields" selected by the user */
info.cmpd_listv = (const struct H5LD_memb_t **)g_listv;
if(g_label) info.cmpd_name = "%s=";
info.line_pre = " %s ";
info.line_cont = " %s ";
info.str_repeat = 8;
}
/* Floating point types should display full precision */
sprintf(fmt_float, "%%1.%dg", FLT_DIG);
info.fmt_float = fmt_float;
sprintf(fmt_double, "%%1.%dg", DBL_DIG);
info.fmt_double = fmt_double;
info.dset_format = "DSET-%s ";
info.dset_hidefileno = 0;
info.obj_format = "-%lu:"H5_PRINTF_HADDR_FMT;
info.obj_hidefileno = 0;
info.dset_blockformat_pre = "%sBlk%lu: ";
info.dset_ptformat_pre = "%sPt%lu: ";
info.line_indent = "";
if(g_display_hex) {
/* Print all data in hexadecimal format if the `-x' or `--hexdump'
* command line switch was given. */
info.raw = TRUE;
}
/* Print the values. */
if((ret_value = h5tools_dump_dset(stdout, &info, &ctx, did, -1, &subset)) < 0)
error_msg("unable to print data\n");
HDfprintf(stdout, "\n");
return(ret_value);
} /* doprint() */
/*
* Verify that messages are moved forward into a "continuation message":
* Create an object header with several continuation chunks
* Remove a message in the last chunk
* The remaining message(s) in the last chunk should be moved forward into the continuation message
* The process will repeat when the continuation message is big enough to hold all the
* messages in the last chunk.
* Result: the number of chunks should be reduced
*/
static herr_t
test_cont(char *filename, hid_t fapl)
{
hid_t file=-1;
H5F_t *f = NULL;
H5O_hdr_info_t hdr_info;
H5O_loc_t oh_locA, oh_locB;
time_t time_new;
const char *short_name = "T";
const char *long_name = "This is the message";
size_t nchunks;
TESTING("object header continuation block");
HDmemset(&oh_locA, 0, sizeof(oh_locA));
HDmemset(&oh_locB, 0, sizeof(oh_locB));
/* Create the file to operate on */
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
FAIL_STACK_ERROR
if(NULL == (f = (H5F_t *)H5I_object(file)))
FAIL_STACK_ERROR
if (H5AC_ignore_tags(f) < 0) {
H5_FAILED();
H5Eprint2(H5E_DEFAULT, stdout);
goto error;
} /* end if */
if(H5O_create(f, H5AC_ind_read_dxpl_id, (size_t)H5O_MIN_SIZE, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_locA/*out*/) < 0)
FAIL_STACK_ERROR
if(H5O_create(f, H5AC_ind_read_dxpl_id, (size_t)H5O_MIN_SIZE, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_locB/*out*/) < 0)
FAIL_STACK_ERROR
time_new = 11111111;
if(H5O_msg_create(&oh_locA, H5O_NAME_ID, 0, 0, &long_name, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_msg_create(&oh_locA, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_msg_create(&oh_locA, H5O_NAME_ID, 0, 0, &short_name, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(1 != H5O_link(&oh_locA, 1, H5AC_ind_read_dxpl_id))
FAIL_STACK_ERROR
if(1 != H5O_link(&oh_locB, 1, H5AC_ind_read_dxpl_id))
FAIL_STACK_ERROR
if(H5AC_flush(f, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_expunge_chunks_test(&oh_locA, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_get_hdr_info(&oh_locA, H5AC_ind_read_dxpl_id, &hdr_info) < 0)
FAIL_STACK_ERROR
nchunks = hdr_info.nchunks;
/* remove the 1st H5O_NAME_ID message */
if(H5O_msg_remove(&oh_locA, H5O_NAME_ID, 0, FALSE, H5AC_ind_read_dxpl_id) < 0)
FAIL_STACK_ERROR
if(H5O_get_hdr_info(&oh_locA, H5AC_ind_read_dxpl_id, &hdr_info) < 0)
FAIL_STACK_ERROR
if(hdr_info.nchunks >= nchunks)
TEST_ERROR
if(H5O_close(&oh_locA, NULL) < 0)
FAIL_STACK_ERROR
if(H5O_close(&oh_locB, NULL) < 0)
FAIL_STACK_ERROR
if(H5Fclose(file) < 0)
FAIL_STACK_ERROR
PASSED();
return SUCCEED;
error:
H5E_BEGIN_TRY {
H5O_close(&oh_locA, NULL);
H5O_close(&oh_locB, NULL);
H5Fclose(file);
} H5E_END_TRY;
return FAIL;
} /* end test_cont() */
/*-------------------------------------------------------------------------
* Function: check_dataset
*
* Purpose: To check whether a dataset can be monitored:
A chunked dataset with unlimited or max. dimension setting
*
* Return: Non-negative on success: dataset can be monitored
* Negative on failure: dataset cannot be monitored
*
* Programmer: Vailin Choi; August 2010
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
check_dataset(hid_t fid, char *dsetname)
{
hid_t did=-1; /* Dataset id */
hid_t dcp=-1; /* Dataset creation property */
hid_t sid=-1; /* Dataset's dataspace id */
int ndims; /* # of dimensions in the dataspace */
unsigned u; /* Local index variable */
hsize_t cur_dims[H5S_MAX_RANK]; /* size of dataspace dimensions */
hsize_t max_dims[H5S_MAX_RANK]; /* maximum size of dataspace dimensions */
hbool_t unlim_max_dims = FALSE; /* whether dataset has unlimited or max. dimension setting */
void *edata;
H5E_auto2_t func;
herr_t ret_value = SUCCEED; /* Return value */
/* Disable error reporting */
H5Eget_auto2(H5E_DEFAULT, &func, &edata);
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
/* Open the dataset */
if((did = H5Dopen2(fid, dsetname, H5P_DEFAULT)) < 0) {
error_msg("unable to open dataset \"%s\"\n", dsetname);
ret_value = FAIL;
goto done;
}
/* Get dataset's creation property list */
if((dcp = H5Dget_create_plist(did)) < 0) {
error_msg("unable to get dataset's creation property list\"%s\"\n", dsetname);
ret_value = FAIL;
goto done;
}
/* Query dataset's layout; the layout should be chunked */
if(H5Pget_layout(dcp) != H5D_CHUNKED) {
error_msg("\"%s\" should be a chunked dataset\n", dsetname);
ret_value = FAIL;
goto done;
}
HDmemset(cur_dims, 0, sizeof cur_dims);
HDmemset(max_dims, 0, sizeof max_dims);
/* Get dataset's dataspace */
if((sid = H5Dget_space(did)) < 0) {
error_msg("can't get dataset's dataspace\"%s\"\n", dsetname);
ret_value = FAIL;
goto done;
}
/* Get dimension size of dataset's dataspace */
if((ndims = H5Sget_simple_extent_dims(sid, cur_dims, max_dims)) < 0) {
error_msg("can't get dataspace dimensions for dataset \"%s\"\n", dsetname);
ret_value = FAIL;
goto done;
}
/* Check whether dataset has unlimited dimension or max. dimension setting */
for(u = 0; u < (unsigned)ndims; u++)
if(max_dims[u] == H5S_UNLIMITED || cur_dims[u] != max_dims[u]) {
unlim_max_dims = TRUE;
break;
}
if(!unlim_max_dims) {
error_msg("\"%s\" should have unlimited or max. dimension setting\n", dsetname);
ret_value = FAIL;
}
done:
H5Eset_auto2(H5E_DEFAULT, func, edata);
/* Closing */
H5E_BEGIN_TRY
H5Sclose(sid);
H5Pclose(dcp);
H5Dclose(did);
H5E_END_TRY
return(ret_value);
} /* check_dataset() */
/*
* Function: parse_command_line
* Purpose: Parse the command line options and return a STRUCT OPTIONS
* structure which will need to be freed by the calling function.
* Return: Pointer to an OPTIONS structure
* Programmer: Bill Wendling, 31. October 2001
* Modifications:
* Added multidimensional testing (Christian Chilan, April, 2008)
*/
static struct options *
parse_command_line(int argc, char *argv[])
{
int opt;
struct options *cl_opts;
int i, default_rank, actual_rank, ranks[4];
cl_opts = (struct options *)HDmalloc(sizeof(struct options));
cl_opts->page_buffer_size = 0;
cl_opts->page_size = 0;
cl_opts->output_file = NULL;
cl_opts->io_types = 0; /* will set default after parsing options */
cl_opts->num_iters = 1;
default_rank = 2;
cl_opts->dset_rank = 0;
cl_opts->buf_rank = 0;
cl_opts->chk_rank = 0;
cl_opts->order_rank = 0;
for(i = 0; i < MAX_DIMS; i++) {
cl_opts->buf_size[i] = (size_t)((i + 1) * 10);
cl_opts->dset_size[i] = (hsize_t)((i + 1) * 100);
cl_opts->chk_size[i] = (size_t)((i + 1) * 10);
cl_opts->order[i] = i + 1;
}
cl_opts->vfd = sec2;
cl_opts->print_times = FALSE; /* Printing times is off by default */
cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */
cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */
cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */
cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */
cl_opts->h5_write_only = FALSE; /* Do both read and write by default */
cl_opts->h5_extendable = FALSE; /* Use extendable dataset */
cl_opts->verify = FALSE; /* No Verify data correctness by default */
while ((opt = get_option(argc, (const char **)argv, s_opts, l_opts)) != EOF) {
switch ((char)opt) {
case 'a':
cl_opts->h5_alignment = parse_size_directive(opt_arg);
break;
case 'G':
cl_opts->page_size = parse_size_directive(opt_arg);
break;
case 'b':
cl_opts->page_buffer_size = parse_size_directive(opt_arg);
break;
case 'A':
{
const char *end = opt_arg;
while (end && *end != '\0') {
char buf[10];
HDmemset(buf, '\0', sizeof(buf));
for (i = 0; *end != '\0' && *end != ','; ++end)
if (isalnum(*end) && i < 10)
buf[i++] = *end;
if (!HDstrcasecmp(buf, "hdf5")) {
cl_opts->io_types |= SIO_HDF5;
} else if (!HDstrcasecmp(buf, "posix")) {
cl_opts->io_types |= SIO_POSIX;
} else {
fprintf(stderr, "sio_perf: invalid --api option %s\n",
buf);
exit(EXIT_FAILURE);
}
if (*end == '\0')
break;
end++;
}
}
break;
#if 0
case 'b':
/* the future "binary" option */
break;
#endif /* 0 */
case 'c':
/* Turn on chunked HDF5 dataset creation */
cl_opts->h5_use_chunks = 1;
{
const char *end = opt_arg;
int j = 0;
while (end && *end != '\0') {
char buf[10];
HDmemset(buf, '\0', sizeof(buf));
for (i = 0; *end != '\0' && *end != ','; ++end)
if (isalnum(*end) && i < 10)
buf[i++] = *end;
cl_opts->chk_size[j] = parse_size_directive(buf);
j++;
if (*end == '\0')
break;
end++;
}
cl_opts->chk_rank = j;
}
break;
case 'D':
{
const char *end = opt_arg;
while (end && *end != '\0') {
char buf[10];
HDmemset(buf, '\0', sizeof(buf));
for (i = 0; *end != '\0' && *end != ','; ++end)
if (isalnum(*end) && i < 10)
buf[i++] = *end;
if (strlen(buf) > 1 || isdigit(buf[0])) {
size_t j;
for (j = 0; j < 10 && buf[j] != '\0'; ++j)
if (!isdigit(buf[j])) {
fprintf(stderr, "sio_perf: invalid --debug option %s\n",
buf);
exit(EXIT_FAILURE);
}
sio_debug_level = atoi(buf);
if (sio_debug_level > 4)
sio_debug_level = 4;
else if (sio_debug_level < 0)
sio_debug_level = 0;
} else {
switch (*buf) {
case 'r':
/* Turn on raw data throughput info */
cl_opts->print_raw = TRUE;
break;
case 't':
/* Turn on time printing */
cl_opts->print_times = TRUE;
break;
case 'v':
/* Turn on verify data correctness*/
cl_opts->verify = TRUE;
break;
default:
fprintf(stderr, "sio_perf: invalid --debug option %s\n", buf);
exit(EXIT_FAILURE);
}
}
if (*end == '\0')
break;
end++;
}
}
break;
case 'e':
{
const char *end = opt_arg;
int j = 0;
while (end && *end != '\0') {
char buf[10];
HDmemset(buf, '\0', sizeof(buf));
for (i = 0; *end != '\0' && *end != ','; ++end)
if (isalnum(*end) && i < 10)
buf[i++] = *end;
cl_opts->dset_size[j] = parse_size_directive(buf);
j++;
if (*end == '\0')
break;
end++;
}
cl_opts->dset_rank = j;
}
break;
case 'i':
cl_opts->num_iters = atoi(opt_arg);
break;
case 'o':
cl_opts->output_file = opt_arg;
break;
case 'T':
cl_opts->h5_threshold = parse_size_directive(opt_arg);
break;
case 'v':
if (!HDstrcasecmp(opt_arg, "sec2")) {
cl_opts->vfd=sec2;
} else if (!HDstrcasecmp(opt_arg, "stdio")) {
cl_opts->vfd=stdio;
} else if (!HDstrcasecmp(opt_arg, "core")) {
cl_opts->vfd=core;
} else if (!HDstrcasecmp(opt_arg, "split")) {
cl_opts->vfd=split;
} else if (!HDstrcasecmp(opt_arg, "multi")) {
cl_opts->vfd=multi;
} else if (!HDstrcasecmp(opt_arg, "family")) {
cl_opts->vfd=family;
} else if (!HDstrcasecmp(opt_arg, "direct")) {
cl_opts->vfd=direct;
} else {
fprintf(stderr, "sio_perf: invalid --api option %s\n",
opt_arg);
exit(EXIT_FAILURE);
}
break;
case 'w':
cl_opts->h5_write_only = TRUE;
break;
case 't':
cl_opts->h5_extendable = TRUE;
break;
case 'x':
{
const char *end = opt_arg;
int j = 0;
while (end && *end != '\0') {
char buf[10];
HDmemset(buf, '\0', sizeof(buf));
for (i = 0; *end != '\0' && *end != ','; ++end)
if (isalnum(*end) && i < 10)
buf[i++] = *end;
cl_opts->buf_size[j] = parse_size_directive(buf);
j++;
if (*end == '\0')
break;
end++;
}
cl_opts->buf_rank = j;
}
break;
case 'r':
{
const char *end = opt_arg;
int j = 0;
while (end && *end != '\0') {
char buf[10];
HDmemset(buf, '\0', sizeof(buf));
for (i = 0; *end != '\0' && *end != ','; ++end)
if (isalnum(*end) && i < 10)
buf[i++] = *end;
cl_opts->order[j] = (int)parse_size_directive(buf);
j++;
if (*end == '\0')
break;
end++;
}
cl_opts->order_rank = j;
}
break;
case 'h':
case '?':
default:
usage(progname);
free(cl_opts);
return NULL;
}
}
/* perform rank consistency analysis */
actual_rank = 0;
ranks[0] = cl_opts->dset_rank;
ranks[1] = cl_opts->buf_rank;
ranks[2] = cl_opts->order_rank;
ranks[3] = cl_opts->chk_rank;
for (i=0; i<4; i++) {
if (ranks[i]>0) {
if (!actual_rank) {
actual_rank = ranks[i];
}
else {
if (actual_rank != ranks[i])
exit(EXIT_FAILURE);
}
}
}
if (!actual_rank)
actual_rank = default_rank;
cl_opts->dset_rank = actual_rank;
cl_opts->buf_rank = actual_rank;
cl_opts->order_rank = actual_rank;
cl_opts->chk_rank = actual_rank;
for (i=0; i<actual_rank; i++) {
if (cl_opts->order[i] > actual_rank) {
exit(EXIT_FAILURE);
}
}
/* set default if none specified yet */
if (!cl_opts->io_types)
cl_opts->io_types = SIO_HDF5 | SIO_POSIX; /* run all API */
/* verify parameters sanity. Adjust if needed. */
/* cap xfer_size with bytes per process */
if (cl_opts->num_iters <= 0)
cl_opts->num_iters = 1;
return cl_opts;
}
/* Test with chunked and partially written SDS. This routine creates a
* "Chunked Not Empty" SDS and writes some chunks but not all to it. It will
* then call SDgetdatasize to verify the sizes.
*/
static intn test_chunked_partial(int32 fid)
{
int32 sds_id, sds_index;
int32 dim_sizes[RANK], origin[RANK];
HDF_CHUNK_DEF c_def; /* Chunking definitions */
int32 flag; /* Chunking flag */
int16 fill_value = 0; /* Fill value */
intn status;
int num_errs = 0; /* number of errors so far */
/* Declare chunks data type and initialize some of them. */
int16 chunk1[CHK_X][CHK_Y] = { {1, 1},
{1, 1},
{1, 1} };
int16 chunk3[CHK_X][CHK_Y] = { {3, 3},
{3, 3},
{3, 3} };
/* Initialize chunk size */
HDmemset(&c_def, 0, sizeof(c_def)) ;
c_def.chunk_lengths[0] = CHK_X;
c_def.chunk_lengths[1] = CHK_Y;
/* Create Y_LENGTH2 x X_LENGTH2 SDS */
dim_sizes[0] = X_LENGTH2;
dim_sizes[1] = Y_LENGTH2;
sds_id = SDcreate(fid, "Chunked Not Empty", DFNT_INT16, RANK, dim_sizes);
CHECK(sds_id, FAIL, "test_chunked_partial: SDcreate 'Chunked Not Empty'");
/* Fill the SDS array with the fill value */
status = SDsetfillvalue(sds_id, (VOIDP)&fill_value);
CHECK(status, FAIL, "test_chunked_partial: SDsetfillvalue");
/* Set info for chunking */
flag = HDF_CHUNK;
status = SDsetchunk(sds_id, c_def, flag);
CHECK(status, FAIL, "test_chunked_partial: SDsetchunk");
/* Write partially to 'Chunked Not Empty' and check the sizes */
/* Write the chunk with the coordinates (0,0) */
origin[0] = 0;
origin[1] = 0;
status = SDwritechunk(sds_id, origin, (VOIDP) chunk1);
CHECK(status, FAIL, "test_chunked_partial: SDwritechunk");
/* Write the chunk with the coordinates (1,0) */
origin[0] = 1;
origin[1] = 0;
status = SDwritechunk(sds_id, origin, (VOIDP) chunk3);
CHECK(status, FAIL, "test_chunked_partial: SDwritechunk");
/* Terminate access to the "Chunked Not Empty" dataset */
status = SDendaccess(sds_id);
CHECK(status, FAIL, "test_chunked_partial: SDendaccess 'Chunked Not Empty'");
/* Check the size of the data of this SDS - only chunked, not compressed,
so both values should be the same; there are two chunks of size
CHK_X*CHK_Y in type int16 written */
check_datasizes(fid, "Chunked Not Empty", CHK_X*CHK_Y*SIZE_INT16*2, CHK_X*CHK_Y*SIZE_INT16*2, &num_errs);
/* Return the number of errors that's been kept track of so far */
return num_errs;
} /* test_chunked_partial */
