int
main (void)
{
hid_t vfile, file, src_space, mem_space, vspace,
vdset, dset; /* Handles */
hid_t dcpl, dapl;
herr_t status;
hsize_t vdsdims[3] = {4*DIM0_1, VDSDIM1, VDSDIM2},
vdsdims_max[3] = {VDSDIM0, VDSDIM1, VDSDIM2},
dims[3] = {DIM0_1, DIM1, DIM2},
memdims[3] = {DIM0_1, DIM1, DIM2},
extdims[3] = {0, DIM1, DIM2}, /* Dimensions of the extended source datasets */
chunk_dims[3] = {DIM0_1, DIM1, DIM2},
dims_max[3] = {DIM0, DIM1, DIM2},
vdsdims_out[3],
vdsdims_max_out[3],
start[3], /* Hyperslab parameters */
stride[3],
count[3],
src_count[3],
block[3];
hsize_t start_out[3], /* Hyperslab parameter out */
stride_out[3],
count_out[3],
block_out[3];
int i, j;
H5D_layout_t layout; /* Storage layout */
size_t num_map; /* Number of mappings */
ssize_t len; /* Length of the string; also a return value */
char *filename;
char *dsetname;
int wdata[DIM0_1*DIM1*DIM2];
/*
* Create source files and datasets. This step is optional.
*/
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < DIM0_1*DIM1*DIM2; j++) wdata[j] = i+1;
/*
* Create the source files and datasets. Write data to each dataset and
* close all resources.
*/
file = H5Fcreate (SRC_FILE[i], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
src_space = H5Screate_simple (RANK, dims, dims_max);
dcpl = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk (dcpl, RANK, chunk_dims);
dset = H5Dcreate2 (file, SRC_DATASET[i], H5T_NATIVE_INT, src_space, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Dwrite (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
status = H5Dclose (dset);
status = H5Fclose (file);
}
vfile = H5Fcreate (VFILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/* Create VDS dataspace. */
vspace = H5Screate_simple (RANK, vdsdims, vdsdims_max);
/* Create dataspaces for the source dataset. */
src_space = H5Screate_simple (RANK, dims, dims_max);
/* Create VDS creation property */
dcpl = H5Pcreate (H5P_DATASET_CREATE);
/* Initialize hyperslab values */
start[0] = 0;
start[1] = 0;
start[2] = 0;
stride[0] = PLANE_STRIDE; /* we will select every fifth plane in VDS */
stride[1] = 1;
stride[2] = 1;
count[0] = H5S_UNLIMITED;
count[1] = 1;
count[2] = 1;
src_count[0] = H5S_UNLIMITED;
src_count[1] = 1;
src_count[2] = 1;
block[0] = 1;
block[1] = DIM1;
block[2] = DIM2;
/*
* Build the mappings
*
*/
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, src_count, block);
for (i=0; i < PLANE_STRIDE; i++) {
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, stride, count, block);
status = H5Pset_virtual (dcpl, vspace, SRC_FILE[i], SRC_DATASET[i], src_space);
start[0]++;
}
H5Sselect_none(vspace);
/* Create a virtual dataset */
vdset = H5Dcreate2 (vfile, DATASET, H5T_NATIVE_INT, vspace, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Sclose (vspace);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
/* Let's add data to the source datasets and check new dimensions for VDS */
/* We will add only one plane to the first source dataset, two planes to the
second one, three to the third, and four to the forth. */
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < (i+1)*DIM1*DIM2; j++) wdata[j] = 10*(i+1);
/*
* Open the source files and datasets. Appen data to each dataset and
* close all resources.
*/
file = H5Fopen (SRC_FILE[i], H5F_ACC_RDWR, H5P_DEFAULT);
dset = H5Dopen2 (file, SRC_DATASET[i], H5P_DEFAULT);
extdims[0] = DIM0_1+i+1;
status = H5Dset_extent (dset, extdims);
src_space = H5Dget_space (dset);
start[0] = DIM0_1;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = 1;
count[2] = 1;
block[0] = i+1;
block[1] = DIM1;
block[2] = DIM2;
memdims[0] = i+1;
mem_space = H5Screate_simple(RANK, memdims, NULL);
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, count, block);
status = H5Dwrite (dset, H5T_NATIVE_INT, mem_space, src_space, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Dclose (dset);
status = H5Fclose (file);
}
status = H5Dclose (vdset);
status = H5Fclose (vfile);
/*
* Now we begin the read section of this example.
*/
/*
* Open file and dataset using the default properties.
*/
vfile = H5Fopen (VFILE, H5F_ACC_RDONLY, H5P_DEFAULT);
/*
* Open VDS using different access properties to use max or
* min extents depending on the sizes of the underlying datasets
*/
dapl = H5Pcreate (H5P_DATASET_ACCESS);
for(i = 0; i < 2; i++) {
status = H5Pset_virtual_view (dapl, i ? H5D_VDS_LAST_AVAILABLE : H5D_VDS_FIRST_MISSING);
vdset = H5Dopen2 (vfile, DATASET, dapl);
/* Let's get space of the VDS and its dimension; we should get 32(or 20)x10x10 */
vspace = H5Dget_space (vdset);
H5Sget_simple_extent_dims (vspace, vdsdims_out, vdsdims_max_out);
printf ("VDS dimensions, bounds = H5D_VDS_%s: ", i ? "LAST_AVAILABLE" : "FIRST_MISSING");
for (j=0; j<RANK; j++)
printf (" %d ", (int)vdsdims_out[j]);
printf ("\n");
/* Close */
status = H5Dclose (vdset);
status = H5Sclose (vspace);
}
status = H5Pclose (dapl);
vdset = H5Dopen2 (vfile, DATASET, H5P_DEFAULT);
/*
* Get creation property list and mapping properties.
*/
dcpl = H5Dget_create_plist (vdset);
/*
* Get storage layout.
*/
layout = H5Pget_layout (dcpl);
if (H5D_VIRTUAL == layout)
printf(" Dataset has a virtual layout \n");
else
printf(" Wrong layout found \n");
/*
* Find number of mappings.
*/
status = H5Pget_virtual_count (dcpl, &num_map);
printf(" Number of mappings is %lu\n", (unsigned long)num_map);
/*
* Get mapping parameters for each mapping.
*/
for (i = 0; i < (int)num_map; i++) {
printf(" Mapping %d \n", i);
printf(" Selection in the virtual dataset \n");
/* Get selection in the virttual dataset */
vspace = H5Pget_virtual_vspace (dcpl, (size_t)i);
if (H5Sget_select_type(vspace) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(vspace)) {
status = H5Sget_regular_hyperslab (vspace, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
/* Get source file name */
len = H5Pget_virtual_filename (dcpl, (size_t)i, NULL, 0);
filename = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_filename (dcpl, (size_t)i, filename, len+1);
printf(" Source filename %s\n", filename);
/* Get source dataset name */
len = H5Pget_virtual_dsetname (dcpl, (size_t)i, NULL, 0);
dsetname = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_dsetname (dcpl, (size_t)i, dsetname, len+1);
printf(" Source dataset name %s\n", dsetname);
/* Get selection in the source dataset */
printf(" Selection in the source dataset \n");
src_space = H5Pget_virtual_srcspace (dcpl, (size_t)i);
if (H5Sget_select_type(src_space) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(src_space)) {
status = H5Sget_regular_hyperslab (src_space, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
H5Sclose(vspace);
H5Sclose(src_space);
free(filename);
free(dsetname);
}
/*
* Close and release resources.
*/
status = H5Pclose (dcpl);
status = H5Dclose (vdset);
status = H5Fclose (vfile);
return 0;
}
/*-------------------------------------------------------------------------
* Function: test_filters_for_datasets
*
* Purpose: Tests creating datasets and writing data with dynamically
* loaded filters
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 14 March 2013
*
*-------------------------------------------------------------------------
*/
static herr_t
test_filters_for_datasets(hid_t file)
{
hid_t dc; /* Dataset creation property list ID */
const hsize_t chunk_size[2] = {FILTER_CHUNK_DIM1, FILTER_CHUNK_DIM2}; /* Chunk dimensions */
unsigned int compress_level = 9;
/*----------------------------------------------------------
* STEP 1: Test deflation by itself.
*----------------------------------------------------------
*/
#ifdef H5_HAVE_FILTER_DEFLATE
puts("Testing deflate filter");
if((dc = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error;
if(H5Pset_chunk (dc, 2, chunk_size) < 0) goto error;
if(H5Pset_deflate (dc, 6) < 0) goto error;
if(test_filter_internal(file,DSET_DEFLATE_NAME,dc) < 0) goto error;
/* Clean up objects used for this test */
if(H5Pclose (dc) < 0) goto error;
#else /* H5_HAVE_FILTER_DEFLATE */
TESTING("deflate filter");
SKIPPED();
puts(" Deflate filter not enabled");
#endif /* H5_HAVE_FILTER_DEFLATE */
/*----------------------------------------------------------
* STEP 2: Test DYNLIB1 by itself.
*----------------------------------------------------------
*/
puts("Testing DYNLIB1 filter");
if((dc = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error;
if(H5Pset_chunk (dc, 2, chunk_size) < 0) goto error;
if(H5Pset_filter (dc, H5Z_FILTER_DYNLIB1, H5Z_FLAG_MANDATORY, (size_t)1, &compress_level) < 0) goto error;
if(test_filter_internal(file,DSET_DYNLIB1_NAME,dc) < 0) goto error;
/* Clean up objects used for this test */
if(H5Pclose (dc) < 0) goto error;
/* Unregister the dynamic filter DYNLIB1 for testing purpose. The next time when this test is run for
* the new file format, the library's H5PL code has to search in the table of loaded plugin libraries
* for this filter. */
if(H5Zunregister(H5Z_FILTER_DYNLIB1) < 0) goto error;
/*----------------------------------------------------------
* STEP 3: Test DYNLIB2 by itself.
*----------------------------------------------------------
*/
puts("Testing DYNLIB2 filter");
if((dc = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error;
if(H5Pset_chunk (dc, 2, chunk_size) < 0) goto error;
if(H5Pset_filter (dc, H5Z_FILTER_DYNLIB2, H5Z_FLAG_MANDATORY, 0, NULL) < 0) goto error;
if(test_filter_internal(file,DSET_DYNLIB2_NAME,dc) < 0) goto error;
/* Clean up objects used for this test */
if(H5Pclose (dc) < 0) goto error;
/* Unregister the dynamic filter DYNLIB2 for testing purpose. The next time when this test is run for
* the new file format, the library's H5PL code has to search in the table of loaded plugin libraries
* for this filter. */
if(H5Zunregister(H5Z_FILTER_DYNLIB2) < 0) goto error;
return 0;
error:
return -1;
}
herr_t H5ARRAYmake( hid_t loc_id,
const char *dset_name,
const char *obversion,
const int rank,
const hsize_t *dims,
int extdim,
hid_t type_id,
hsize_t *dims_chunk,
void *fill_data,
int compress,
char *complib,
int shuffle,
int fletcher32,
const void *data)
{
hid_t dataset_id, space_id;
hsize_t *maxdims = NULL;
hid_t plist_id = 0;
unsigned int cd_values[6];
int chunked = 0;
int i;
/* Check whether the array has to be chunked or not */
if (dims_chunk) {
chunked = 1;
}
if(chunked) {
maxdims = malloc(rank*sizeof(hsize_t));
if(!maxdims) return -1;
for(i=0;i<rank;i++) {
if (i == extdim) {
maxdims[i] = H5S_UNLIMITED;
}
else {
maxdims[i] = dims[i] < dims_chunk[i] ? dims_chunk[i] : dims[i];
}
}
}
/* Create the data space for the dataset. */
if ( (space_id = H5Screate_simple( rank, dims, maxdims )) < 0 )
return -1;
if (chunked) {
/* Modify dataset creation properties, i.e. enable chunking */
plist_id = H5Pcreate (H5P_DATASET_CREATE);
if ( H5Pset_chunk ( plist_id, rank, dims_chunk ) < 0 )
return -1;
/* Set the fill value using a struct as the data type. */
if (fill_data) {
if ( H5Pset_fill_value( plist_id, type_id, fill_data ) < 0 )
return -1;
}
else {
if ( H5Pset_fill_time(plist_id, H5D_FILL_TIME_ALLOC) < 0 )
return -1;
}
/*
Dataset creation property list is modified to use
*/
/* Fletcher must be first */
if (fletcher32) {
if ( H5Pset_fletcher32( plist_id) < 0 )
return -1;
}
/* Then shuffle (not if blosc is activated) */
if ((shuffle) && (strcmp(complib, "blosc") != 0)) {
if ( H5Pset_shuffle( plist_id) < 0 )
return -1;
}
/* Finally compression */
if (compress) {
cd_values[0] = compress;
cd_values[1] = (int)(atof(obversion) * 10);
if (extdim <0)
cd_values[2] = CArray;
else
cd_values[2] = EArray;
/* The default compressor in HDF5 (zlib) */
if (strcmp(complib, "zlib") == 0) {
if ( H5Pset_deflate( plist_id, compress) < 0 )
return -1;
}
/* The Blosc compressor does accept parameters */
else if (strcmp(complib, "blosc") == 0) {
cd_values[4] = compress;
cd_values[5] = shuffle;
if ( H5Pset_filter( plist_id, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 6, cd_values) < 0 )
return -1;
}
/* The LZO compressor does accept parameters */
else if (strcmp(complib, "lzo") == 0) {
if ( H5Pset_filter( plist_id, FILTER_LZO, H5Z_FLAG_OPTIONAL, 3, cd_values) < 0 )
return -1;
}
/* The bzip2 compress does accept parameters */
else if (strcmp(complib, "bzip2") == 0) {
if ( H5Pset_filter( plist_id, FILTER_BZIP2, H5Z_FLAG_OPTIONAL, 3, cd_values) < 0 )
return -1;
}
else {
/* Compression library not supported */
fprintf(stderr, "Compression library not supported\n");
return -1;
}
}
/* Create the (chunked) dataset */
if ((dataset_id = H5Dcreate(loc_id, dset_name, type_id,
space_id, plist_id )) < 0 )
goto out;
}
else { /* Not chunked case */
/* Create the dataset. */
if ((dataset_id = H5Dcreate(loc_id, dset_name, type_id,
space_id, H5P_DEFAULT )) < 0 )
goto out;
}
/* Write the dataset only if there is data to write */
if (data)
{
if ( H5Dwrite( dataset_id, type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, data ) < 0 )
goto out;
}
/* Terminate access to the data space. */
if ( H5Sclose( space_id ) < 0 )
return -1;
/* End access to the property list */
if (plist_id)
if ( H5Pclose( plist_id ) < 0 )
goto out;
/* Release resources */
if (maxdims)
free(maxdims);
return dataset_id;
out:
H5Dclose( dataset_id );
H5Sclose( space_id );
if (maxdims)
free(maxdims);
if (dims_chunk)
free(dims_chunk);
return -1;
}
void hdfFileSaver::saveNow(void)
{
QDir prnt;
herr_t status;
hsize_t maxdims[1];
hsize_t d_dims[1];
int rank = 1;
hsize_t datasetlength;
size_t memdims[1];
hsize_t count[1]; /* size of subset in the file */
hsize_t offset[1]; /* subset offset in the file */
hsize_t mcount[1];
hsize_t moffset[1];
hid_t prop;
if (events==0)
return;
if (access(fname.toStdString().c_str(), F_OK) ==-1)
{
hfile = H5Fcreate(
fname.toStdString().c_str(),
H5F_ACC_EXCL ,
H5P_DEFAULT,
H5P_DEFAULT ) ;
topgroup = H5Gcreate(
hfile,
"iqdata_raw",
H5P_DEFAULT,
H5P_DEFAULT,
H5P_DEFAULT);
hid_t aid3 = H5Screate(H5S_SCALAR);
hid_t atype = H5Tcopy(H5T_C_S1);
H5Tset_size(atype, 4);
hid_t attr3 = H5Acreate1(topgroup, "type", atype, aid3, H5P_DEFAULT);
status = H5Awrite(attr3, atype,"dict");
H5Aclose(attr3);
H5Tclose(atype);
H5Sclose(aid3);
}
else
{
//printf("File already exists, will append\n");
hfile = H5Fopen(
fname.toStdString().c_str(),
H5F_ACC_RDWR,
H5P_DEFAULT );
topgroup = H5Gopen(hfile, "iqdata_raw", H5P_DEFAULT);
//fprintf(stderr,"Bad Hdf file already existant, cannot open\n");
}
if (hfile <=0 || topgroup <=0)
{
fprintf(stderr,"Bad Hdf file, cannot open\n");
return;
}
if (true)
{
//QHash<int,QHash<QString,QList<float> > > events;
QList<int> keys1=(*events).keys();
for (int i=0;i<keys1.length();i++)
{
int chan = keys1[i];
if ((*events)[chan]["bin"].length() > 0)
{
int bin = (int)((*events)[chan]["bin"][0]);
QString dirname = QString("keyint_%1").arg(chan);
//turn off errors when we query the group, using open
hid_t error_stack = H5Eget_current_stack();
H5E_auto2_t oldfunc;
void *old_client_data;
H5Eget_auto(error_stack, &oldfunc, &old_client_data);
H5Eset_auto(error_stack, NULL, NULL);
channelgroup = H5Gopen(
topgroup,
dirname.toStdString().c_str(),
H5P_DEFAULT);
//turn errors back on.
H5Eset_auto(error_stack, oldfunc, old_client_data);
if (channelgroup<0)
{
channelgroup = H5Gcreate(
topgroup,
dirname.toStdString().c_str(),
H5P_DEFAULT,
H5P_DEFAULT,
H5P_DEFAULT);
hid_t aid3 = H5Screate(H5S_SCALAR);
hid_t atype = H5Tcopy(H5T_C_S1);
H5Tset_size(atype, 4);
hid_t attr3 = H5Acreate1(channelgroup, "type", atype, aid3, H5P_DEFAULT);
status = H5Awrite(attr3, atype,"dict");
H5Aclose(attr3);
H5Tclose(atype);
H5Sclose(aid3);
}
if (channelgroup!=0)
{
QList<QString> keys2=(*events)[keys1[i]].keys();
for (int i2=0;i2<keys2.length();i2++)
{
QString dname = QString("keystr_%1").arg(keys2[i2]);
datasetlength = (*events)[keys1[i]][keys2[i2]].length();
if (datasetlength>0)
{
//Try to open dataset if it exists
//turn off errors when we query the group, using open
hid_t error_stack = H5Eget_current_stack();
H5E_auto2_t oldfunc;
void *old_client_data;
H5Eget_auto(error_stack, &oldfunc, &old_client_data);
H5Eset_auto(error_stack, NULL, NULL);
//query or open dataset
curdataset = H5Dopen(channelgroup,dname.toStdString().c_str(),H5P_DEFAULT);
//turn errors back on.
H5Eset_auto(error_stack, oldfunc, old_client_data);
//if cannot open dataset, create it, and make it chunked.
if (curdataset<=0)
{
//set up size info, chunks etc...
maxdims[0]=H5S_UNLIMITED;
rank = 1;
d_dims[0]=datasetlength;
curdataspace= H5Screate_simple(rank, d_dims,maxdims);
prop = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk(prop, rank, d_dims);
if (status) trap();
curdataset = H5Dcreate(
channelgroup,
dname.toStdString().c_str(),
H5T_NATIVE_FLOAT,
curdataspace,
H5P_DEFAULT,
prop,
H5P_DEFAULT);
hid_t aid3 = H5Screate(H5S_SCALAR);
hid_t atype = H5Tcopy(H5T_C_S1);
H5Tset_size(atype, 5);
hid_t attr3 = H5Acreate1(curdataset, "type", atype, aid3, H5P_DEFAULT);
status = H5Awrite(attr3, atype,"array");
H5Aclose(attr3);
H5Tclose(atype);
H5Sclose(aid3);
H5Pclose(prop);
}//if (curdataset<=0)
else
{
//get dataspace from exant dataset
curdataspace = H5Dget_space(curdataset);
H5Sget_simple_extent_dims(curdataspace, d_dims, maxdims);
H5Sclose(curdataspace);
//if dataset already exist, extend its size
d_dims[0] = d_dims[0] + datasetlength;
status = H5Dset_extent(curdataset, d_dims);
if (status) trap();
curdataspace = H5Dget_space(curdataset);
H5Sget_simple_extent_dims(curdataspace, d_dims, maxdims);
}//else
if (curdataset>0)
{
//
// Set up stride, offset, count block in dastasets...
//
// stride[0]=1;//hop size between floats in dataset...
//block[0]=1;//get 1 slab
count[0]=datasetlength;//size of hyperslab or chunk of data
offset[0]=d_dims[0] - datasetlength;//offset from beginning of file.
status = H5Sselect_hyperslab(
curdataspace,
H5S_SELECT_SET,
offset,
NULL,
count,
NULL);
if (status) trap();
rank = 1;
memdims[0]=datasetlength;
memdataspace= H5Screate_simple(rank, (const hsize_t*)memdims,NULL);
mcount[0] = datasetlength;
moffset[0] = 0;
status = H5Sselect_hyperslab(
memdataspace,
H5S_SELECT_SET,
moffset,
NULL,
mcount,
NULL);
if (status) trap();
for (int mmm=0;mmm<datasetlength;mmm++)
this->rawdata[mmm]=(*events)[keys1[i]][keys2[i2]][mmm];
status = H5Dwrite(
curdataset,
H5T_NATIVE_FLOAT,
memdataspace,
curdataspace,
H5P_DEFAULT,
this->rawdata);
if (status) trap();
H5Sclose(memdataspace);
H5Sclose(curdataspace);
H5Dclose(curdataset);
} //if (curdataset>0)
}
}//for (int i2=0;i2<keys2.length();i2++)
status = H5Gclose(channelgroup);
}//if (channelgroup!=0)
}//if ((*events)[chan]["bin"].length() > 0)
}//for (int i=0;i<keys1.length();i++)
}//if (true)
status = H5Gclose(topgroup);
status = H5Fclose(hfile);
}
int main(int argc, char* argv[])
{
char c;
int ix, iy, iz, i;
MPI_Comm mpicomm;
MPI_Info mpiinfo;
int mpirank;
int mpisize;
double *data3d, *data2d, *x, *y, *z, t;
int localx, localy, localwidth, localheight;
int maxwidth, maxheight;
const char* filename = "output.h5";
hid_t fileid, plist, filespace, memspace, dimvar, varid;
hsize_t size[NDIMS], maxsize[NDIMS], chunksize[NDIMS];
hsize_t start[NDIMS], count[NDIMS];
char varname[32];
mpicomm = MPI_COMM_WORLD;
mpiinfo = MPI_INFO_NULL;
MPI_Init(&argc, &argv);
MPI_Comm_size(mpicomm, &mpisize);
MPI_Comm_rank(mpicomm, &mpirank);
if(! mpirank) printf("Creating some data...\n");
// Distribute our data values in a pism-y way
GetLocalBounds(XSIZE, YSIZE, mpirank, mpisize,
&localx, &localy, &localwidth, &localheight);
printf("Rank%02d: x=%d, y=%d, width=%d, height=%d\n",
mpirank, localx, localy, localwidth, localheight);
data2d = (double*)malloc(localwidth * localheight * sizeof(double));
data3d = (double*)malloc(localwidth * localheight * ZSIZE * sizeof(double));
x = (double*)malloc(localwidth * sizeof(double));
y = (double*)malloc(localheight * sizeof(double));
z = (double*)malloc(ZSIZE * sizeof(double));
t = 0.0;
for(ix = 0; ix < localwidth; ix++)
{
x[ix] = ix + localx;
for(iy = 0; iy < localheight; iy++)
{
y[iy] = iy + localy;
data2d[ix*localheight + iy] = (ix+localx)*localheight + iy+localy;
for(iz = 0; iz < ZSIZE; iz++)
{
z[iz] = iz;
data3d[ix*localheight*ZSIZE + iy*ZSIZE + iz] =
(ix+localx)*YSIZE*ZSIZE + (iy+localy)*ZSIZE + iz;
}
}
}
if(! mpirank) printf("Creating HDF5 file...\n");
plist = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(plist, mpicomm, mpiinfo);
// TODO: this seems like a good place to put optimizations, and indeed
// PISM is adding several additional properties, like setting block sizes,
// cache eviction policies, fs striping parameters, etc.
fileid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, plist);
H5Pclose(plist);
if(! mpirank) printf("Setting up dimensions...\n");
if(! mpirank) printf("Creating time dimension...\n");
// Define the time dimension
size[0] = 1;
maxsize[0] = H5S_UNLIMITED;
chunksize[0] = 1;
filespace = H5Screate_simple(1, size, maxsize);
plist = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(plist, 1, chunksize); // It is strictly required to set chunksize when using
// the low-level api. Contiguous datasets are not allowed
// to use the unlimited dimension.
dimvar = H5Dcreate(fileid, TNAME, H5T_NATIVE_DOUBLE, filespace,
H5P_DEFAULT, plist, H5P_DEFAULT);
H5Pclose(plist);
H5DSset_scale(dimvar, TNAME);
H5Dclose(dimvar);
H5Sclose(filespace);
#ifdef OLD_WRITE_PATTERN
if(! mpirank) printf("Writing time dimension...\n");
dimvar = H5Dopen(fileid, TNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, size, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE); // TODO: Pism does this, but comments suggest it is questionable
start[0] = 0;
count[0] = 1;
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, &t);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
#endif
if(! mpirank) printf("Creating x dimension...\n");
size[0] = XSIZE;
filespace = H5Screate_simple(1, size, 0);
dimvar = H5Dcreate(fileid, XNAME, H5T_NATIVE_DOUBLE, filespace,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5DSset_scale(dimvar, XNAME);
H5Dclose(dimvar);
H5Sclose(filespace);
#ifdef OLD_WRITE_PATTERN
if(! mpirank) printf("Writing x dimension...\n");
dimvar = H5Dopen(fileid, XNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, size, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
start[0] = 0;
count[0] = XSIZE;
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, x);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
#endif
if(! mpirank) printf("Creating y dimension...\n");
size[0] = YSIZE;
filespace = H5Screate_simple(1, size, 0);
dimvar = H5Dcreate(fileid, YNAME, H5T_NATIVE_DOUBLE, filespace,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5DSset_scale(dimvar, YNAME);
H5Dclose(dimvar);
H5Sclose(filespace);
#ifdef OLD_WRITE_PATTERN
if(! mpirank) printf("Writing y dimension...\n");
dimvar = H5Dopen(fileid, YNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, size, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
start[0] = 0;
count[0] = YSIZE;
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, y);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
#endif
if(! mpirank) printf("Creating z dimension...\n");
size[0] = ZSIZE;
filespace = H5Screate_simple(1, size, 0);
dimvar = H5Dcreate(fileid, ZNAME, H5T_NATIVE_DOUBLE, filespace,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5DSset_scale(dimvar, ZNAME);
H5Dclose(dimvar);
H5Sclose(filespace);
#ifdef OLD_WRITE_PATTERN
if(! mpirank) printf("Writing z dimension...\n");
dimvar = H5Dopen(fileid, ZNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, size, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
start[0] = 0;
count[0] = ZSIZE;
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, z);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
#endif
if(! mpirank) printf("Defining variables...\n");
MPI_Allreduce(&localwidth, &maxwidth, 1, MPI_INT, MPI_MAX, mpicomm);
MPI_Allreduce(&localheight, &maxheight, 1, MPI_INT, MPI_MAX, mpicomm);
size[TDIM] = 1;
size[XDIM] = XSIZE;
size[YDIM] = YSIZE;
size[ZDIM] = ZSIZE;
maxsize[TDIM] = H5S_UNLIMITED;
maxsize[XDIM] = XSIZE;
maxsize[YDIM] = YSIZE;
maxsize[ZDIM] = ZSIZE;
chunksize[TDIM] = 1;
chunksize[XDIM] = maxwidth;
chunksize[YDIM] = maxheight;
chunksize[ZDIM] = ZSIZE; // Looks like pism might use 1 here...
for(i = 0; i < NVARS; i++)
{
sprintf(varname, "var3d-%02d", i);
plist = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(plist, NDIMS, chunksize);
filespace = H5Screate_simple(NDIMS, size, maxsize);
varid = H5Dcreate(fileid, varname, H5T_NATIVE_DOUBLE, filespace,
H5P_DEFAULT, plist, H5P_DEFAULT);
H5Pclose(plist);
H5Sclose(filespace);
H5Dclose(varid);
sprintf(varname, "var2d-%02d", i);
plist = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(plist, NDIMS-1, chunksize);
filespace = H5Screate_simple(NDIMS-1, size, maxsize);
varid = H5Dcreate(fileid, varname, H5T_NATIVE_DOUBLE, filespace,
H5P_DEFAULT, plist, H5P_DEFAULT);
H5Pclose(plist);
H5Sclose(filespace);
H5Dclose(varid);
}
#ifndef OLD_WRITE_PATTERN
if(! mpirank) printf("Writing time dimension...\n");
start[0] = 0;
count[0] = 1;
dimvar = H5Dopen(fileid, TNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, count, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE); // TODO: Pism does this, but comments suggest it is questionable
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, &t);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
if(! mpirank) printf("Writing x dimension...\n");
start[0] = 0;
count[0] = XSIZE;
dimvar = H5Dopen(fileid, XNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, count, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, x);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
if(! mpirank) printf("Writing y dimension...\n");
start[0] = 0;
count[0] = YSIZE;
dimvar = H5Dopen(fileid, YNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, count, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, y);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
if(! mpirank) printf("Writing z dimension...\n");
start[0] = 0;
count[0] = ZSIZE;
dimvar = H5Dopen(fileid, ZNAME, H5P_DEFAULT);
filespace = H5Dget_space(dimvar);
memspace = H5Screate_simple(1, count, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(dimvar, H5T_NATIVE_DOUBLE, memspace, filespace, plist, z);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(dimvar);
#endif
if(! mpirank) printf("Writing variable data...\n");
for(i = 0; i < NVARS; i++)
{
sprintf(varname, "var3d-%02d", i);
if(! mpirank) printf("Writing %s...\n", varname);
start[TDIM] = 0;
start[XDIM] = localx;
start[YDIM] = localy;
start[ZDIM] = 0;
count[TDIM] = 1;
count[XDIM] = localwidth;
count[YDIM] = localheight;
count[ZDIM] = ZSIZE;
varid = H5Dopen(fileid, varname, H5P_DEFAULT);
filespace = H5Dget_space(varid);
memspace = H5Screate_simple(NDIMS, count, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(varid, H5T_NATIVE_DOUBLE, memspace, filespace, plist, data3d);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(varid);
sprintf(varname, "var2d-%02d", i);
if(! mpirank) printf("Writing %s...\n", varname);
start[TDIM] = 0;
start[XDIM] = localx;
start[YDIM] = localy;
count[TDIM] = 1;
count[XDIM] = localwidth;
count[YDIM] = localheight;
varid = H5Dopen(fileid, varname, H5P_DEFAULT);
filespace = H5Dget_space(varid);
memspace = H5Screate_simple(NDIMS-1, count, 0);
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, 0, count, 0);
H5Dwrite(varid, H5T_NATIVE_DOUBLE, memspace, filespace, plist, data2d);
H5Pclose(plist);
H5Sclose(filespace);
H5Sclose(memspace);
H5Dclose(varid);
}
if(! mpirank) printf("Closing file...\n");
H5Fclose(fileid);
if(! mpirank) printf("Done.\n");
free(data2d);
free(data3d);
free(x);
free(y);
free(z);
MPI_Finalize();
return 0;
}
AccessTraceWriter::AccessTraceWriter(g_string _fname, uint32_t numChildren) : fname(_fname) {
// Create record structure
hid_t accType = H5Tenum_create(H5T_NATIVE_USHORT);
uint16_t val;
H5Tenum_insert(accType, "GETS", (val=GETS,&val));
H5Tenum_insert(accType, "GETX", (val=GETX,&val));
H5Tenum_insert(accType, "PUTS", (val=PUTS,&val));
H5Tenum_insert(accType, "PUTX", (val=PUTX,&val));
size_t offset = 0;
size_t size = H5Tget_size(H5T_NATIVE_ULONG)*2 + H5Tget_size(H5T_NATIVE_UINT) + H5Tget_size(H5T_NATIVE_USHORT) + H5Tget_size(accType);
hid_t recType = H5Tcreate(H5T_COMPOUND, size);
auto insertType = [&](const char* name, hid_t type) {
H5Tinsert(recType, name, offset, type);
offset += H5Tget_size(type);
};
insertType("lineAddr", H5T_NATIVE_ULONG);
insertType("cycle", H5T_NATIVE_ULONG);
insertType("lat", H5T_NATIVE_UINT);
insertType("childId", H5T_NATIVE_USHORT);
insertType("accType", accType);
hid_t fid = H5Fcreate(fname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
if (fid == H5I_INVALID_HID) panic("Could not create HDF5 file %s", fname.c_str());
// HACK: We want to use the SHUF filter... create the raw dataset instead of the packet table
// hid_t table = H5PTcreate_fl(fid, "accs", recType, PT_CHUNKSIZE, 9);
// if (table == H5I_INVALID_HID) panic("Could not create HDF5 packet table");
hsize_t dims[1] = {0};
hsize_t dims_chunk[1] = {PT_CHUNKSIZE};
hsize_t maxdims[1] = {H5S_UNLIMITED};
hid_t space_id = H5Screate_simple(1, dims, maxdims);
hid_t plist_id = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(plist_id, 1, dims_chunk);
H5Pset_shuffle(plist_id);
H5Pset_deflate(plist_id, 9);
hid_t table = H5Dcreate2(fid, "accs", recType, space_id, H5P_DEFAULT, plist_id, H5P_DEFAULT);
if (table == H5I_INVALID_HID) panic("Could not create HDF5 dataset");
H5Dclose(table);
// info("%ld %ld %ld %ld", sizeof(PackedAccessRecord), size, offset, H5Tget_size(recType));
assert(offset == size);
assert(size == sizeof(PackedAccessRecord));
hid_t ncAttr = H5Acreate2(fid, "numChildren", H5T_NATIVE_UINT, H5Screate(H5S_SCALAR), H5P_DEFAULT, H5P_DEFAULT);
H5Awrite(ncAttr, H5T_NATIVE_UINT, &numChildren);
H5Aclose(ncAttr);
hid_t fAttr = H5Acreate2(fid, "finished", H5T_NATIVE_UINT, H5Screate(H5S_SCALAR), H5P_DEFAULT, H5P_DEFAULT);
uint32_t finished = 0;
H5Awrite(fAttr, H5T_NATIVE_UINT, &finished);
H5Aclose(fAttr);
H5Fclose(fid);
// Initialize buffer
buf = gm_calloc<PackedAccessRecord>(PT_CHUNKSIZE);
cur = 0;
max = PT_CHUNKSIZE;
assert((uint32_t)(((char*) &buf[1]) - ((char*) &buf[0])) == sizeof(PackedAccessRecord));
}
/*+++++++++++++++++++++++++
.IDENTifer PYTABLE_make_array
.PURPOSE create extensible HDF5 dataset
.INPUT/OUTPUT
call as stat = PYTABLE_make_array( locID, dset_name, title, rank, dims,
extdim, typeID, dims_chunk, fill_data,
compress, shuffle, fletcher32, buff );
input:
hid_t locID : HDF5 identifier of file or group
char *dset_name : name of dataset
char *title :
int rank : number of dimensions
hsize_t *dims : size of each dimension
int extdim : index of expendable dimension
hid_t typeID : data type (HDF5 identifier)
hsize_t *dims_chunk : chunk sizes
void *fill_data : Fill value for data
unsigned int compress : compression level (zero for no compression)
bool shuffle : shuffel data for better compression
bool fletcher32 :
void *buffer : buffer with data to write (or NULL)
.RETURNS A negative value is returned on failure.
.COMMENTS none
-------------------------*/
herr_t PYTABLE_make_array( hid_t locID, const char *dset_name,
const char *title, const int rank,
const hsize_t *dims, int extdim, hid_t typeID,
const hsize_t *dims_chunk, void *fill_data,
unsigned int compress, bool shuffle,
bool fletcher32, const void *buffer )
{
register int ni;
hid_t dataID = -1, spaceID = -1;
herr_t stat;
/* check if the array has to be chunked or not */
if ( dims_chunk != NULL ) {
hid_t plistID;
hsize_t *maxdims = (hsize_t *) malloc( rank * sizeof(hsize_t) );
if ( maxdims == NULL )
NADC_GOTO_ERROR( NADC_ERR_ALLOC, "maxdims" );
for ( ni = 0; ni < rank; ni++ ) {
if ( ni == extdim )
maxdims[ni] = H5S_UNLIMITED;
else
maxdims[ni] =
dims[ni] < dims_chunk[ni] ? dims_chunk[ni] : dims[ni];
}
spaceID = H5Screate_simple( rank, dims, maxdims );
free( maxdims );
if ( spaceID < 0 ) NADC_GOTO_ERROR( NADC_ERR_HDF_SPACE, "" );
/* Modify dataset creation properties, i.e. enable chunking */
plistID = H5Pcreate( H5P_DATASET_CREATE );
if ( H5Pset_chunk( plistID, rank, dims_chunk ) < 0 ) goto done;
/* set the fill value using a struct as the data type */
if ( fill_data != NULL
&& H5Pset_fill_value( plistID, typeID, fill_data ) < 0 )
goto done;
/* dataset creation property list is modified to use */
/* fletcher must be first */
if ( fletcher32 ) {
if ( H5Pset_fletcher32( plistID ) < 0 ) goto done;
}
/* then shuffle */
if ( shuffle ) {
if ( H5Pset_shuffle( plistID ) < 0 ) goto done;
}
/* finally compression */
if ( compress > 0 ) {
if ( H5Pset_deflate( plistID, compress ) < 0 ) goto done;
}
/* create the (chunked) dataset */
dataID = H5Dcreate( locID, dset_name, typeID, spaceID,
H5P_DEFAULT, plistID, H5P_DEFAULT );
if ( dataID < 0 )
NADC_GOTO_ERROR( NADC_ERR_HDF_DATA, dset_name );
/* end access to the property list */
if ( H5Pclose( plistID ) < 0 ) goto done;
} else {
spaceID = H5Screate_simple( rank, dims, NULL );
if ( spaceID < 0 ) return -1;
/* create the dataset (not chunked) */
dataID = H5Dcreate( locID, dset_name, typeID, spaceID,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT );
if ( dataID < 0 )
NADC_GOTO_ERROR( NADC_ERR_HDF_DATA, dset_name );
}
/*
* write the data
*/
stat = H5Dwrite( dataID, typeID, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer );
if ( stat < 0 ) NADC_GOTO_ERROR( NADC_ERR_HDF_WR, "" );
(void) H5Dclose( dataID );
(void) H5Sclose( spaceID );
/*
* Set the conforming array attributes
*
* attach the CLASS attribute
*/
(void) H5LTset_attribute_string( locID, dset_name, "CLASS",
PY_ARRAY_CLASS );
/* attach the EXTDIM attribute in case of enlargeable arrays */
(void) H5LTset_attribute_int( locID, dset_name, "EXTDIM", &extdim, 1 );
/* attach the FLAVOR attribute */
(void) H5LTset_attribute_string( locID, dset_name, "FLAVOR",
PY_ARRAY_FLAVOR );
/* attach the VERSION attribute */
(void) H5LTset_attribute_string( locID, dset_name, "VERSION",
PY_ARRAY_VERSION );
/* attach the TITLE attribute */
(void) H5LTset_attribute_string( locID, dset_name, "TITLE", title );
return 0;
done:
if ( dataID > 0 ) (void) H5Dclose( dataID );
if ( spaceID > 0 ) (void) H5Sclose( spaceID );
return -1;
}
void RegionalSummary::writeH5(hid_t &file_id, string group_name) {
herr_t status;
hid_t group_id = H5CreateOrOpenGroup(file_id, group_name);
hid_t dataset_id;
hid_t dataspace_id;
hsize_t h5_dims[1];
// region_origin
vector<unsigned int> coord_buf(2,0);
coord_buf[0] = origin_.first;
coord_buf[1] = origin_.second;
h5_dims[0] = 2;
dataspace_id = H5Screate_simple (1, h5_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "region_origin", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &coord_buf[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// region_dim
coord_buf[0] = dim_.first;
coord_buf[1] = dim_.second;
h5_dims[0] = 2;
dataspace_id = H5Screate_simple (1, h5_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "region_dim", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &coord_buf[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// n_err_
h5_dims[0] = 1;
dataspace_id = H5Screate_simple (1, h5_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "n_err", H5T_NATIVE_UINT_LEAST64, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT_LEAST64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &n_err_);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// n_aligned_
h5_dims[0] = 1;
dataspace_id = H5Screate_simple (1, h5_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "n_aligned", H5T_NATIVE_UINT_LEAST64, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT_LEAST64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &n_aligned_);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// data_dim
AssertDims();
coord_buf[0] = n_flow_;
coord_buf[1] = max_hp_;
h5_dims[0] = 2;
dataspace_id = H5Screate_simple (1, h5_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "data_dim", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &coord_buf[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// hp_count_ and hp_err_
hsize_t dims[2];
dims[0] = n_flow_;
dims[1] = max_hp_;
vector<uint64_t> data_buf;
hsize_t cdims[2];
hid_t plist_id;
// hp_count_
LoadDataBuffer(max_hp_,n_flow_,data_buf,hp_count_);
dataspace_id = H5Screate_simple (2, dims, NULL);
plist_id = H5Pcreate (H5P_DATASET_CREATE);
cdims[0] = min(n_flow_,(unsigned int) 20);
cdims[1] = min(max_hp_,(unsigned int) 20);
status = H5Pset_chunk (plist_id, 2, cdims);
status = H5Pset_deflate (plist_id, 9);
dataset_id = H5Dcreate2 (group_id, "hp_count", H5T_NATIVE_UINT_LEAST64, dataspace_id, H5P_DEFAULT, plist_id, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT_LEAST64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data_buf[0]);
status = H5Dclose (dataset_id);
status = H5Pclose (plist_id);
status = H5Sclose (dataspace_id);
// hp_err_
LoadDataBuffer(max_hp_,n_flow_,data_buf,hp_err_);
dataspace_id = H5Screate_simple (2, dims, NULL);
plist_id = H5Pcreate (H5P_DATASET_CREATE);
cdims[0] = min(n_flow_,(unsigned int) 20);
cdims[1] = min(max_hp_,(unsigned int) 20);
status = H5Pset_chunk (plist_id, 2, cdims);
status = H5Pset_deflate (plist_id, 9);
dataset_id = H5Dcreate2 (group_id, "hp_err", H5T_NATIVE_UINT_LEAST64, dataspace_id, H5P_DEFAULT, plist_id, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT_LEAST64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data_buf[0]);
status = H5Dclose (dataset_id);
status = H5Pclose (plist_id);
status = H5Sclose (dataspace_id);
status = H5Gclose (group_id);
}
void ErrorData::writeH5(hid_t &file_id, string group_name) {
herr_t status;
hid_t group_id = H5CreateOrOpenGroup(file_id, group_name);
hid_t dataset_id;
hid_t dataspace_id;
vector<unsigned int> buf32;
// Write the region origin & dimensions
buf32.resize(2);
hsize_t region_dims[1];
// origin
buf32[0] = region_origin_.first;
buf32[1] = region_origin_.second;
region_dims[0] = 2;
dataspace_id = H5Screate_simple (1, region_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "region_origin", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buf32[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// dim
buf32[0] = region_dim_.first;
buf32[1] = region_dim_.second;
region_dims[0] = 2;
dataspace_id = H5Screate_simple (1, region_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "region_dim", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buf32[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// error_data_dim
unsigned int n_row=5;
unsigned int n_col=ins_.Size();
buf32[0] = n_row;
buf32[1] = n_col;
region_dims[0] = 2;
dataspace_id = H5Screate_simple (1, region_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "error_data_dim", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buf32[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// make buffer for Nx5 matrix of error data to write
vector<uint64_t> buf64;
LoadErrorDataBuffer(n_col,n_row,buf64);
// Create compressed dataset
hsize_t dims[2];
dims[0] = n_row;
dims[1] = n_col;
dataspace_id = H5Screate_simple (2, dims, NULL);
hid_t plist_id = H5Pcreate (H5P_DATASET_CREATE);
hsize_t cdims[2];
cdims[0] = min(n_row,(unsigned int) 20);
cdims[1] = min(n_col,(unsigned int) 100);
status = H5Pset_chunk (plist_id, 2, cdims);
status = H5Pset_deflate (plist_id, 9);
dataset_id = H5Dcreate2 (group_id, "error_data", H5T_NATIVE_UINT_LEAST64, dataspace_id, H5P_DEFAULT, plist_id, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT_LEAST64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buf64[0]);
status = H5Dclose (dataset_id);
status = H5Pclose (plist_id);
status = H5Sclose (dataspace_id);
status = H5Gclose (group_id);
}
void HpData::writeH5(hid_t &file_id, string group_name) {
herr_t status;
hid_t group_id = H5CreateOrOpenGroup(file_id, group_name);
hid_t dataset_id;
hid_t dataspace_id;
vector<unsigned int> coord_buf(2,0);
// Write the region origin & dimensions
hsize_t region_dims[1];
// origin
coord_buf[0] = origin_.first;
coord_buf[1] = origin_.second;
region_dims[0] = 2;
dataspace_id = H5Screate_simple (1, region_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "region_origin", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &coord_buf[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// dim
coord_buf[0] = dim_.first;
coord_buf[1] = dim_.second;
region_dims[0] = 2;
dataspace_id = H5Screate_simple (1, region_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "region_dim", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &coord_buf[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
unsigned int n_row = 0;
unsigned int n_col = 0;
if(hp_count_.begin() != hp_count_.end()) {
n_row = hp_count_.begin()->second.size();
n_col = (n_row > 0) ? hp_count_.begin()->second[0].size() : 0;
}
// dim
coord_buf[0] = n_row;
coord_buf[1] = n_col;
region_dims[0] = 2;
dataspace_id = H5Screate_simple (1, region_dims, NULL);
dataset_id = H5Dcreate2 (group_id, "hp_data_dim", H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &coord_buf[0]);
status = H5Dclose (dataset_id);
status = H5Sclose (dataspace_id);
// now write the per-nuc error tables
vector<uint64_t> hp_table_buf;
map< char, vector< vector<uint64_t> > >::iterator it;
for(it = hp_count_.begin(); it != hp_count_.end(); ++it) {
string nuc;
stringstream ss;
ss << it->first;
ss >> nuc;
LoadHpDataBuffer(n_col,n_row,hp_table_buf,it->second);
// Create compressed dataset
hsize_t dims[2];
dims[0] = n_row;
dims[1] = n_col;
dataspace_id = H5Screate_simple (2, dims, NULL);
hid_t plist_id = H5Pcreate (H5P_DATASET_CREATE);
hsize_t cdims[2];
cdims[0] = min(n_row,(unsigned int) 20);
cdims[1] = min(n_col,(unsigned int) 20);
status = H5Pset_chunk (plist_id, 2, cdims);
status = H5Pset_deflate (plist_id, 9);
dataset_id = H5Dcreate2 (group_id, nuc.c_str(), H5T_NATIVE_UINT_LEAST64, dataspace_id, H5P_DEFAULT, plist_id, H5P_DEFAULT);
status = H5Dwrite (dataset_id, H5T_NATIVE_UINT_LEAST64, H5S_ALL, H5S_ALL, H5P_DEFAULT, &hp_table_buf[0]);
status = H5Dclose (dataset_id);
status = H5Pclose (plist_id);
status = H5Sclose (dataspace_id);
}
status = H5Gclose (group_id);
}
void pyne::Material::write_hdf5(std::string filename, std::string datapath,
std::string nucpath, float row, int chunksize) {
int row_num = (int) row;
// Turn off annoying HDF5 errors
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
//Set file access properties so it closes cleanly
hid_t fapl;
fapl = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fclose_degree(fapl,H5F_CLOSE_STRONG);
// Create new/open datafile.
hid_t db;
if (pyne::file_exists(filename)) {
bool ish5 = H5Fis_hdf5(filename.c_str());
if (!ish5)
throw h5wrap::FileNotHDF5(filename);
db = H5Fopen(filename.c_str(), H5F_ACC_RDWR, fapl);
}
else
db = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
//
// Read in nuclist if available, write it out if not
//
bool nucpath_exists = h5wrap::path_exists(db, nucpath);
std::vector<int> nuclides;
int nuc_size;
hsize_t nuc_dims[1];
if (nucpath_exists) {
nuclides = h5wrap::h5_array_to_cpp_vector_1d<int>(db, nucpath, H5T_NATIVE_INT);
nuc_size = nuclides.size();
nuc_dims[0] = nuc_size;
} else {
nuclides = std::vector<int>();
for (pyne::comp_iter i = comp.begin(); i != comp.end(); i++)
nuclides.push_back(i->first);
nuc_size = nuclides.size();
// Create the data if it doesn't exist
int nuc_data [nuc_size];
for (int n = 0; n != nuc_size; n++)
nuc_data[n] = nuclides[n];
nuc_dims[0] = nuc_size;
hid_t nuc_space = H5Screate_simple(1, nuc_dims, NULL);
hid_t nuc_set = H5Dcreate2(db, nucpath.c_str(), H5T_NATIVE_INT, nuc_space,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite(nuc_set, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, nuc_data);
H5Fflush(db, H5F_SCOPE_GLOBAL);
};
//
// Write out the data itself to the file
//
hid_t data_set, data_space, data_hyperslab;
int data_rank = 1;
hsize_t data_dims[1] = {1};
hsize_t data_max_dims[1] = {H5S_UNLIMITED};
hsize_t data_offset[1] = {0};
size_t material_struct_size = sizeof(pyne::material_struct) + sizeof(double)*nuc_size;
hid_t desc = H5Tcreate(H5T_COMPOUND, material_struct_size);
hid_t comp_values_array_type = H5Tarray_create2(H5T_NATIVE_DOUBLE, 1, nuc_dims);
// make the data table type
H5Tinsert(desc, "mass", HOFFSET(pyne::material_struct, mass), H5T_NATIVE_DOUBLE);
H5Tinsert(desc, "density", HOFFSET(pyne::material_struct, density),
H5T_NATIVE_DOUBLE);
H5Tinsert(desc, "atoms_per_molecule", HOFFSET(pyne::material_struct, atoms_per_mol),
H5T_NATIVE_DOUBLE);
H5Tinsert(desc, "comp", HOFFSET(pyne::material_struct, comp),
comp_values_array_type);
material_struct * mat_data = new material_struct[material_struct_size];
(*mat_data).mass = mass;
(*mat_data).density = density;
(*mat_data).atoms_per_mol = atoms_per_molecule;
for (int n = 0; n != nuc_size; n++) {
if (0 < comp.count(nuclides[n]))
(*mat_data).comp[n] = comp[nuclides[n]];
else
(*mat_data).comp[n] = 0.0;
};
// get / make the data set
bool datapath_exists = h5wrap::path_exists(db, datapath);
if (datapath_exists) {
data_set = H5Dopen2(db, datapath.c_str(), H5P_DEFAULT);
data_space = H5Dget_space(data_set);
data_rank = H5Sget_simple_extent_dims(data_space, data_dims, data_max_dims);
// Determine the row size.
if (std::signbit(row))
row_num = data_dims[0] + row; // careful, row is negative
if (data_dims[0] <= row_num) {
// row == -0, extend to data set so that we can append, or
// row_num is larger than current dimension, resize to accomodate.
data_dims[0] = row_num + 1;
H5Dset_extent(data_set, data_dims);
}
data_offset[0] = row_num;
} else {
// Get full space
data_space = H5Screate_simple(1, data_dims, data_max_dims);
// Make data set properties to enable chunking
hid_t data_set_params = H5Pcreate(H5P_DATASET_CREATE);
hsize_t chunk_dims[1] ={chunksize};
H5Pset_chunk(data_set_params, 1, chunk_dims);
H5Pset_deflate(data_set_params, 1);
material_struct * data_fill_value = new material_struct[material_struct_size];
(*data_fill_value).mass = -1.0;
(*data_fill_value).density= -1.0;
(*data_fill_value).atoms_per_mol = -1.0;
for (int n = 0; n != nuc_size; n++)
(*data_fill_value).comp[n] = 0.0;
H5Pset_fill_value(data_set_params, desc, &data_fill_value);
// Create the data set
data_set = H5Dcreate2(db, datapath.c_str(), desc, data_space, H5P_DEFAULT,
data_set_params, H5P_DEFAULT);
H5Dset_extent(data_set, data_dims);
// Add attribute pointing to nuc path
hid_t nuc_attr_type = H5Tcopy(H5T_C_S1);
H5Tset_size(nuc_attr_type, nucpath.length());
hid_t nuc_attr_space = H5Screate(H5S_SCALAR);
hid_t nuc_attr = H5Acreate2(data_set, "nucpath", nuc_attr_type, nuc_attr_space,
H5P_DEFAULT, H5P_DEFAULT);
H5Awrite(nuc_attr, nuc_attr_type, nucpath.c_str());
H5Fflush(db, H5F_SCOPE_GLOBAL);
// Remember to de-allocate
delete[] data_fill_value;
};
// Get the data hyperslab
data_hyperslab = H5Dget_space(data_set);
hsize_t data_count[1] = {1};
H5Sselect_hyperslab(data_hyperslab, H5S_SELECT_SET, data_offset, NULL, data_count, NULL);
// Get a memory space for writing
hid_t mem_space = H5Screate_simple(1, data_count, data_max_dims);
// Write the row...
H5Dwrite(data_set, desc, mem_space, data_hyperslab, H5P_DEFAULT, mat_data);
// Close out the Dataset
H5Fflush(db, H5F_SCOPE_GLOBAL);
H5Dclose(data_set);
H5Sclose(data_space);
H5Tclose(desc);
//
// Write out the metadata to the file
//
std::string attrpath = datapath + "_metadata";
hid_t metadatapace, attrtype, metadataet, metadatalab, attrmemspace;
int attrrank;
attrtype = H5Tvlen_create(H5T_NATIVE_CHAR);
// get / make the data set
bool attrpath_exists = h5wrap::path_exists(db, attrpath);
if (attrpath_exists) {
metadataet = H5Dopen2(db, attrpath.c_str(), H5P_DEFAULT);
metadatapace = H5Dget_space(metadataet);
attrrank = H5Sget_simple_extent_dims(metadatapace, data_dims, data_max_dims);
if (data_dims[0] <= row_num) {
// row == -0, extend to data set so that we can append, or
// row_num is larger than current dimension, resize to accomodate.
data_dims[0] = row_num + 1;
H5Dset_extent(metadataet, data_dims);
}
data_offset[0] = row_num;
} else {
hid_t metadataetparams;
hsize_t attrchunkdims [1];
// Make data set properties to enable chunking
metadataetparams = H5Pcreate(H5P_DATASET_CREATE);
attrchunkdims[0] = chunksize;
H5Pset_chunk(metadataetparams, 1, attrchunkdims);
H5Pset_deflate(metadataetparams, 1);
hvl_t attrfillvalue [1];
attrfillvalue[0].len = 3;
attrfillvalue[0].p = (char *) "{}\n";
H5Pset_fill_value(metadataetparams, attrtype, &attrfillvalue);
// make dataset
metadatapace = H5Screate_simple(1, data_dims, data_max_dims);
metadataet = H5Dcreate2(db, attrpath.c_str(), attrtype, metadatapace,
H5P_DEFAULT, metadataetparams, H5P_DEFAULT);
H5Dset_extent(metadataet, data_dims);
};
// set the attr string
hvl_t attrdata [1];
Json::FastWriter writer;
std::string metadatatr = writer.write(metadata);
attrdata[0].p = (char *) metadatatr.c_str();
attrdata[0].len = metadatatr.length();
// write the attr
metadatalab = H5Dget_space(metadataet);
H5Sselect_hyperslab(metadatalab, H5S_SELECT_SET, data_offset, NULL, data_count, NULL);
attrmemspace = H5Screate_simple(1, data_count, data_max_dims);
H5Dwrite(metadataet, attrtype, attrmemspace, metadatalab, H5P_DEFAULT, attrdata);
// close attr data objects
H5Fflush(db, H5F_SCOPE_GLOBAL);
H5Dclose(metadataet);
H5Sclose(metadatapace);
H5Tclose(attrtype);
// Close out the HDF5 file
H5Fclose(db);
// Remember the milk!
// ...by which I mean to deallocate
delete[] mat_data;
};
int
main(void)
{
hid_t faplid = -1; /* file access property list ID (all files) */
hid_t src_sid = -1; /* source dataset's dataspace ID */
hid_t src_dcplid = -1; /* source dataset property list ID */
hid_t vds_sid = -1; /* VDS dataspace ID */
hid_t vds_dcplid = -1; /* VDS dataset property list ID */
hid_t fid = -1; /* HDF5 file ID */
hid_t did = -1; /* dataset ID */
hsize_t start[RANK]; /* starting point for hyperslab */
int map_start = -1; /* starting point in the VDS map */
int i; /* iterator */
/* Start by creating the virtual dataset (VDS) dataspace and creation
* property list. The individual source datasets are then created
* and the VDS map (stored in the VDS property list) is updated.
*/
/* Create VDS dcpl */
if((vds_dcplid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_fill_value(vds_dcplid, VDS_DATATYPE,
&VDS_FILL_VALUE) < 0)
TEST_ERROR
/* Create VDS dataspace */
if((vds_sid = H5Screate_simple(RANK, VDS_DIMS,
VDS_MAX_DIMS)) < 0)
TEST_ERROR
/************************************
* Create source files and datasets *
************************************/
start[0] = 0;
start[1] = 0;
start[2] = 0;
map_start = 0;
/* All SWMR files need to use the latest file format */
if((faplid = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR
if(H5Pset_libver_bounds(faplid, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
TEST_ERROR
for(i = 0; i < N_SOURCES; i++) {
/* source dataset dcpl */
if((src_dcplid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_chunk(src_dcplid, RANK, PLANES[i]) < 0)
TEST_ERROR
if(H5Pset_fill_value(src_dcplid, SOURCE_DATATYPE,
&FILL_VALUES[i]) < 0)
TEST_ERROR
/* Use a mix of compressed and uncompressed datasets */
if(0 != i % 2)
if(H5Pset_deflate(src_dcplid, COMPRESSION_LEVEL) < 0)
TEST_ERROR
/* Create source file, dataspace, and dataset */
if((fid = H5Fcreate(FILE_NAMES[i], H5F_ACC_TRUNC,
H5P_DEFAULT, faplid)) < 0)
TEST_ERROR
if((src_sid = H5Screate_simple(RANK, DIMS[i],
MAX_DIMS[i])) < 0)
TEST_ERROR
if((did = H5Dcreate2(fid, SOURCE_DSET_NAME,
SOURCE_DATATYPE, src_sid,
H5P_DEFAULT, src_dcplid, H5P_DEFAULT)) < 0)
TEST_ERROR
/* set up hyperslabs for source and destination datasets */
start[1] = 0;
if(H5Sselect_hyperslab(src_sid, H5S_SELECT_SET, start, NULL,
MAX_DIMS[i], NULL) < 0)
TEST_ERROR
start[1] = map_start;
if(H5Sselect_hyperslab(vds_sid, H5S_SELECT_SET, start, NULL,
MAX_DIMS[i], NULL) < 0)
TEST_ERROR
map_start += PLANES[i][1];
/* Add VDS mapping */
if(H5Pset_virtual(vds_dcplid, vds_sid, FILE_NAMES[i],
SOURCE_DSET_PATH, src_sid) < 0)
TEST_ERROR
/* close */
if(H5Sclose(src_sid) < 0)
TEST_ERROR
if(H5Pclose(src_dcplid) < 0)
TEST_ERROR
if(H5Dclose(did) < 0)
TEST_ERROR
if(H5Fclose(fid) < 0)
TEST_ERROR
} /* end for */
/*******************
* Create VDS file *
*******************/
/* file */
if((fid = H5Fcreate(VDS_FILE_NAME, H5F_ACC_TRUNC,
H5P_DEFAULT, faplid)) < 0)
TEST_ERROR
/* dataset */
if((did = H5Dcreate2(fid, VDS_DSET_NAME, VDS_DATATYPE, vds_sid,
H5P_DEFAULT, vds_dcplid, H5P_DEFAULT)) < 0)
TEST_ERROR
/* close */
if(H5Pclose(faplid) < 0)
TEST_ERROR
if(H5Pclose(vds_dcplid) < 0)
TEST_ERROR
if(H5Sclose(vds_sid) < 0)
TEST_ERROR
if(H5Dclose(did) < 0)
TEST_ERROR
if(H5Fclose(fid) < 0)
TEST_ERROR
return EXIT_SUCCESS;
error:
H5E_BEGIN_TRY {
if(faplid >= 0)
(void)H5Pclose(faplid);
if(src_sid >= 0)
(void)H5Sclose(src_sid);
if(src_dcplid >= 0)
(void)H5Pclose(src_dcplid);
if(vds_sid >= 0)
(void)H5Sclose(vds_sid);
if(vds_dcplid >= 0)
(void)H5Pclose(vds_dcplid);
if(fid >= 0)
(void)H5Fclose(fid);
if(did >= 0)
(void)H5Dclose(did);
} H5E_END_TRY
return EXIT_FAILURE;
} /* end main */
/*-------------------------------------------------------------------------
* Function: test_dataset_write_with_filters
*
* Purpose: Tests creating datasets and writing data with dynamically loaded filters
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
test_dataset_write_with_filters(hid_t fid)
{
hid_t dcpl_id = -1; /* Dataset creation property list ID */
unsigned int compress_level; /* Deflate compression level */
unsigned int filter1_data; /* Data used by filter 1 */
unsigned int libver_values[4]; /* Used w/ the filter that makes HDF5 calls */
/*----------------------------------------------------------
* STEP 1: Test deflation by itself.
*----------------------------------------------------------
*/
HDputs("Testing dataset writes with deflate filter");
#ifdef H5_HAVE_FILTER_DEFLATE
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl_id, 2, chunk_sizes_g) < 0)
TEST_ERROR;
compress_level = 6;
if (H5Pset_deflate(dcpl_id, compress_level) < 0)
TEST_ERROR;
/* Ensure the filter works */
if (ensure_filter_works(fid, DSET_DEFLATE_NAME, dcpl_id) < 0)
TEST_ERROR;
/* Clean up objects used for this test */
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
#else /* H5_HAVE_FILTER_DEFLATE */
SKIPPED();
HDputs(" Deflate filter not enabled");
#endif /* H5_HAVE_FILTER_DEFLATE */
/*----------------------------------------------------------
* STEP 2: Test filter plugin 1 by itself.
*----------------------------------------------------------
*/
HDputs(" dataset writes with filter plugin 1");
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl_id, 2, chunk_sizes_g) < 0)
TEST_ERROR;
/* Set up the filter, passing in the amount the filter will add and subtract
* from each data element. Note that this value has an arbitrary max of 9.
*/
filter1_data = 9;
if (H5Pset_filter(dcpl_id, FILTER1_ID, H5Z_FLAG_MANDATORY, (size_t)1, &filter1_data) < 0)
TEST_ERROR;
/* Ensure the filter works */
if (ensure_filter_works(fid, DSET_FILTER1_NAME, dcpl_id) < 0)
TEST_ERROR;
/* Clean up objects used for this test */
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
/* Unregister the dynamic filter for testing purpose. The next time when this test is run for
* the new file format, the library's H5PL code has to search in the table of loaded plugin libraries
* for this filter.
*/
if (H5Zunregister(FILTER1_ID) < 0)
TEST_ERROR;
/*----------------------------------------------------------
* STEP 3: Test filter plugin 2 by itself.
*----------------------------------------------------------
*/
HDputs(" dataset writes with filter plugin 2");
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl_id, 2, chunk_sizes_g) < 0)
TEST_ERROR;
if (H5Pset_filter(dcpl_id, FILTER2_ID, H5Z_FLAG_MANDATORY, 0, NULL) < 0)
TEST_ERROR;
/* Ensure the filter works */
if (ensure_filter_works(fid, DSET_FILTER2_NAME, dcpl_id) < 0)
TEST_ERROR;
/* Clean up objects used for this test */
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
/* Unregister the dynamic filter for testing purpose. The next time when this test is run for
* the new file format, the library's H5PL code has to search in the table of loaded plugin libraries
* for this filter.
*/
if (H5Zunregister(FILTER2_ID) < 0)
TEST_ERROR;
/*----------------------------------------------------------
* STEP 4: Test filter plugin 3 by itself.
* (This filter plugin makes HDF5 API calls)
*----------------------------------------------------------
*/
HDputs(" dataset writes with filter plugin 3");
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl_id, 2, chunk_sizes_g) < 0)
TEST_ERROR;
/* Set the add/subtract value for the filter */
libver_values[0] = 9;
/* Get the library bounds and add to the filter data */
if (H5get_libversion(&libver_values[1], &libver_values[2], &libver_values[3]) < 0)
TEST_ERROR;
if (H5Pset_filter(dcpl_id, FILTER3_ID, H5Z_FLAG_MANDATORY, (size_t)4, libver_values) < 0)
TEST_ERROR;
/* Ensure the filter works */
if (ensure_filter_works(fid, DSET_FILTER3_NAME, dcpl_id) < 0)
TEST_ERROR;
/* Clean up objects used for this test */
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
/* Unregister the dynamic filter for testing purpose. The next time when this test is run for
* the new file format, the library's H5PL code has to search in the table of loaded plugin libraries
* for this filter.
*/
if (H5Zunregister(FILTER3_ID) < 0)
TEST_ERROR;
return SUCCEED;
error:
/* Clean up objects used for this test */
H5E_BEGIN_TRY {
H5Pclose(dcpl_id);
} H5E_END_TRY
return FAIL;
} /* end test_dataset_write_with_filters() */
/*-------------------------------------------------------------------------
* Function: test_refresh
*
* Purpose: This function tests refresh (evict/reload) of individual
* objects' metadata from the metadata cache.
*
* Return: 0 on Success, 1 on Failure
*
* Programmer: Mike McGreevy
* August 17, 2010
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t test_refresh(void)
{
/**************************************************************************
*
* Test Description:
*
* This test will build an HDF5 file with several objects in a varying
* hierarchical layout. It will then flush the entire file to disk. Then,
* an attribute will be added to each object in the file.
*
* One by one, this process will flush each object to disk, individually.
* It will also be coordinating with another process, which will open
* the object before it is flushed by this process, and then refresh the
* object after it's been flushed, comparing the before and after object
* information to ensure that they are as expected. (i.e., most notably,
* that an attribute has been added, and is only visible after a
* successful call to a H5*refresh function).
*
* As with the flush case, the implemention is a bit tricky as it's
* dealing with signals going back and forth between the two processes
* to ensure the timing is correct, but basically, an example:
*
* Step 1. Dataset is created.
* Step 2. Dataset is flushed.
* Step 3. Attribute on Dataset is created.
* Step 4. Another process opens the dataset and verifies that it does
* not see an attribute (as the attribute hasn't been flushed yet).
* Step 5. This process flushes the dataset again (with Attribute attached).
* Step 6. The other process calls H5Drefresh, which should evict/reload
* the object's metadata, and thus pick up the attribute that's
* attached to it. Most other before/after object information is
* compared for sanity as well.
* Step 7. Rinse and Repeat for each object in the file.
*
**************************************************************************/
/**************************************************************************
* Generated Test File will look like this:
*
* GROUP "/"
* DATASET "Dataset1"
* GROUP "Group1" {
* DATASET "Dataset2"
* GROUP "Group2" {
* DATATYPE "CommittedDatatype3"
* }
* }
* GROUP "Group3" {
* DATASET "Dataset3"
* DATATYPE "CommittedDatatype2"
* }
* DATATYPE "CommittedDatatype1"
**************************************************************************/
/* Variables */
hid_t aid,fid,sid,tid1,did,dcpl,fapl = 0;
hid_t gid,gid2,gid3,tid2,tid3,did2,did3,status = 0;
hsize_t dims[2] = {50,50};
hsize_t cdims[2] = {1,1};
int fillval = 2;
/* Testing Message */
HDfprintf(stdout, "Testing individual object refresh behavior:\n");
/* Cleanup any old error or signal files */
CLEANUP_FILES;
/* ================ */
/* CREATE TEST FILE */
/* ================ */
/* Create File */
if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR;
if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR;
if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE, H5P_DEFAULT, fapl)) < 0) TEST_ERROR;
/* Create data space and types */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR;
if ( H5Pset_chunk(dcpl, 2, cdims) < 0 ) TEST_ERROR;
if ( H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0 ) TEST_ERROR;
if ((sid = H5Screate_simple(2, dims, dims)) < 0) TEST_ERROR;
if ((tid1 = H5Tcopy(H5T_NATIVE_INT)) < 0) TEST_ERROR;
if ((tid2 = H5Tcopy(H5T_NATIVE_CHAR)) < 0) TEST_ERROR;
if ((tid3 = H5Tcopy(H5T_NATIVE_LONG)) < 0) TEST_ERROR;
/* Create Group1 */
if ((gid = H5Gcreate2(fid, "Group1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create Group2 */
if ((gid2 = H5Gcreate2(gid, "Group2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create Group3 */
if ((gid3 = H5Gcreate2(fid, "Group3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create Dataset1 */
if ((did = H5Dcreate2(fid, "Dataset1", tid1, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create Dataset2 */
if ((did2 = H5Dcreate2(gid, "Dataset2", tid3, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create Dataset3 */
if ((did3 = H5Dcreate2(gid3, "Dataset3", tid2, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create CommittedDatatype1 */
if ((status = H5Tcommit2(fid, "CommittedDatatype1", tid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create CommittedDatatype2 */
if ((status = H5Tcommit2(gid2, "CommittedDatatype2", tid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Create CommittedDatatype3 */
if ((status = H5Tcommit2(gid3, "CommittedDatatype3", tid3, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
/* Flush File to Disk */
if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) TEST_ERROR;
/* Create an attribute on each object. These will not immediately hit disk,
and thus be unavailable to another process until this process flushes
the object and the other process refreshes from disk. */
if ((aid = H5Acreate2(did, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(did2, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(did3, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(gid, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(gid2, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(gid3, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(tid1, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(tid2, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
if ((aid = H5Acreate2(tid3, "Attribute", tid1, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
if (H5Aclose(aid) < 0) TEST_ERROR;
/* ================ */
/* Refresh Datasets */
/* ================ */
TESTING("to ensure that H5Drefresh correctly refreshes single datasets");
/* Verify First Dataset can be refreshed with H5Drefresh */
if (start_refresh_verification_process(D1) != 0) TEST_ERROR;
if (H5Oflush(did) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
/* Verify Second Dataset can be refreshed with H5Drefresh */
if (start_refresh_verification_process(D2) != 0) TEST_ERROR;
if (H5Oflush(did2) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
PASSED();
/* ============== */
/* Refresh Groups */
/* ============== */
TESTING("to ensure that H5Grefresh correctly refreshes single groups");
/* Verify First Group can be refreshed with H5Grefresh */
if (start_refresh_verification_process(G1) != 0) TEST_ERROR;
if (H5Oflush(gid) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
/* Verify Second Group can be refreshed with H5Grefresh */
if (start_refresh_verification_process(G2) != 0) TEST_ERROR;
if (H5Oflush(gid2) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
PASSED();
/* ================= */
/* Refresh Datatypes */
/* ================= */
TESTING("to ensure that H5Trefresh correctly refreshes single datatypes");
/* Verify First Committed Datatype can be refreshed with H5Trefresh */
if (start_refresh_verification_process(T1) != 0) TEST_ERROR;
if (H5Oflush(tid1) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
/* Verify Second Committed Datatype can be refreshed with H5Trefresh */
if (start_refresh_verification_process(T2) != 0) TEST_ERROR;
if (H5Oflush(tid2) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
PASSED();
/* =============== */
/* Refresh Objects */
/* =============== */
TESTING("to ensure that H5Orefresh correctly refreshes single objects");
/* Verify Third Dataset can be refreshed with H5Orefresh */
if (start_refresh_verification_process(D3) != 0) TEST_ERROR;
if (H5Oflush(did3) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
/* Verify Third Group can be refreshed with H5Orefresh */
if (start_refresh_verification_process(G3) != 0) TEST_ERROR;
if (H5Oflush(gid3) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
/* Verify Third Committed Datatype can be refreshed with H5Orefresh */
if (start_refresh_verification_process(T3) != 0) TEST_ERROR;
if (H5Oflush(tid3) < 0) TEST_ERROR;
if (end_refresh_verification_process() != 0) TEST_ERROR;
PASSED();
/* ================== */
/* Cleanup and Return */
/* ================== */
/* Close Stuff */
if (H5Pclose(fapl) < 0) TEST_ERROR;
if (H5Tclose(tid1) < 0) TEST_ERROR;
if (H5Tclose(tid2) < 0) TEST_ERROR;
if (H5Tclose(tid3) < 0) TEST_ERROR;
if (H5Dclose(did) < 0) TEST_ERROR;
if (H5Dclose(did2) < 0) TEST_ERROR;
if (H5Dclose(did3) < 0) TEST_ERROR;
if (H5Gclose(gid) < 0) TEST_ERROR;
if (H5Gclose(gid2) < 0) TEST_ERROR;
if (H5Gclose(gid3) < 0) TEST_ERROR;
if (H5Sclose(sid) < 0) TEST_ERROR;
if (H5Fclose(fid) < 0) TEST_ERROR;
/* Delete Test File */
HDremove(FILENAME);
if (end_verification() < 0) TEST_ERROR;
return SUCCEED;
error:
/* Return */
return FAIL;
} /* test_refresh() */
/*-------------------------------------------------------------------------
* Function: test_skip_compress_write2
*
* Purpose: Test skipping compression filter when there are three filters
* for the dataset
*
* Return: Success: 0
*
* Failure: 1
*
* Programmer: Raymond Lu
* 30 November 2012
*
*-------------------------------------------------------------------------
*/
static int
test_skip_compress_write2(hid_t file)
{
hid_t dataspace = -1, dataset = -1;
hid_t mem_space = -1;
hid_t cparms = -1, dxpl = -1;
hsize_t dims[2] = {NX, NY};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY};
herr_t status;
int i, j, n;
unsigned filter_mask = 0;
int origin_direct_buf[CHUNK_NX][CHUNK_NY];
int direct_buf[CHUNK_NX][CHUNK_NY];
int check_chunk[CHUNK_NX][CHUNK_NY];
hsize_t offset[2] = {0, 0};
size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
int aggression = 9; /* Compression aggression setting */
hsize_t start[2]; /* Start of hyperslab */
hsize_t stride[2]; /* Stride of hyperslab */
hsize_t count[2]; /* Block count */
hsize_t block[2]; /* Block sizes */
TESTING("skipping compression filters but keep two other filters");
/*
* Create the data space with unlimited dimensions.
*/
if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0)
goto error;
if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0)
goto error;
/*
* Modify dataset creation properties, i.e. enable chunking and compression.
* The order of filters is bogus 1 + deflate + bogus 2.
*/
if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0)
goto error;
/* Register and enable first bogus filter */
if(H5Zregister (H5Z_BOGUS1) < 0)
goto error;
if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS1, 0, (size_t)0, NULL) < 0)
goto error;
/* Enable compression filter */
if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0)
goto error;
/* Register and enable second bogus filter */
if(H5Zregister (H5Z_BOGUS2) < 0)
goto error;
if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS2, 0, (size_t)0, NULL) < 0)
goto error;
/*
* Create a new dataset within the file using cparms
* creation properties.
*/
if((dataset = H5Dcreate2(file, DATASETNAME3, H5T_NATIVE_INT, dataspace, H5P_DEFAULT,
cparms, H5P_DEFAULT)) < 0)
goto error;
if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0)
goto error;
/* Initialize data for one chunk. Apply operations of two bogus filters to the chunk */
for(i = n = 0; i < CHUNK_NX; i++)
for(j = 0; j < CHUNK_NY; j++) {
origin_direct_buf[i][j] = n++;
direct_buf[i][j] = (origin_direct_buf[i][j] + ADD_ON) * FACTOR;
}
/* write the uncompressed chunk data repeatedly to dataset, using the direct writing function.
* Indicate skipping the compression filter but keep the other two bogus filters */
offset[0] = CHUNK_NX;
offset[1] = CHUNK_NY;
/* compression filter is the middle one to be skipped */
filter_mask = 0x00000002;
if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0)
goto error;
if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0)
goto error;
if(H5Dclose(dataset) < 0)
goto error;
if((dataset = H5Dopen2(file, DATASETNAME3, H5P_DEFAULT)) < 0)
goto error;
/*
* Select hyperslab for one chunk in the file
*/
start[0] = CHUNK_NX; start[1] = CHUNK_NY;
stride[0] = 1; stride[1] = 1;
count[0] = 1; count[1] = 1;
block[0] = CHUNK_NX; block[1] = CHUNK_NY;
if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0)
goto error;
/* Read the chunk back */
if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0)
goto error;
/* Check that the values read are the same as the values written */
for(i = 0; i < CHUNK_NX; i++) {
for(j = 0; j < CHUNK_NY; j++) {
if(origin_direct_buf[i][j] != check_chunk[i][j]) {
printf(" 1. Read different values than written.");
printf(" At index %d,%d\n", i, j);
printf(" origin_direct_buf=%d, check_chunk=%d\n", origin_direct_buf[i][j], check_chunk[i][j]);
goto error;
}
}
}
/*
* Close/release resources.
*/
H5Dclose(dataset);
H5Sclose(mem_space);
H5Sclose(dataspace);
H5Pclose(cparms);
H5Pclose(dxpl);
PASSED();
return 0;
error:
H5E_BEGIN_TRY {
H5Dclose(dataset);
H5Sclose(mem_space);
H5Sclose(dataspace);
H5Pclose(cparms);
H5Pclose(dxpl);
} H5E_END_TRY;
return 1;
}
int main(int argc, char *argv[])
{
hid_t fid = -1;
hid_t access_plist = -1;
hid_t create_plist = -1;
hid_t cparm = -1;
hid_t datatype = -1;
hid_t dataspace = -1;
hid_t dataset = -1;
hid_t memspace = -1;
hid_t groupDetector = -1;
int rank = 1;
hsize_t chunk[2] = {10,10};
hsize_t dims[2] = {1,1};
hsize_t elementSize[2] = {1,1};
hsize_t maxdims[2] = {H5S_UNLIMITED,H5S_UNLIMITED};
int ivalue[2];
int fillValue = 0;
/* Open the source file and dataset */
/* All SWMR files need to use the latest file format */
access_plist = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fclose_degree(access_plist, H5F_CLOSE_STRONG);
#if H5_VERSION_GE(1,9,178)
H5Pset_object_flush_cb(access_plist, cFlushCallback, NULL);
#endif
H5Pset_libver_bounds(access_plist, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
create_plist = H5Pcreate(H5P_FILE_CREATE);
fid = H5Fcreate("test_string_swmr.h5", H5F_ACC_TRUNC, create_plist, access_plist);
/* Data */
rank = 2;
dims[0] = 10;
dims[1] = 10;
dataspace = H5Screate_simple(rank, dims, dims);
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(cparm, rank, chunk);
datatype = H5Tcopy(H5T_NATIVE_INT8);
H5Pset_fill_value(cparm, datatype, &fillValue);
groupDetector = H5Gcreate(fid, "detector", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
access_plist = H5Pcreate(H5P_DATASET_ACCESS);
H5Pset_chunk_cache(access_plist, 503, 100, 1.0);
dataset = H5Dcreate2(groupDetector, "data1",
datatype, dataspace,
H5P_DEFAULT, cparm, access_plist);
ivalue[0] = 1;
ivalue[1] = 1;
writeInt32Attribute(dataset, "NDArrayDimBinning", 2, ivalue);
ivalue[0] = 0;
ivalue[1] = 0;
writeInt32Attribute(dataset, "NDArrayDimOffset", 2, ivalue);
writeInt32Attribute(dataset, "NDArrayDimReverse", 2, ivalue);
ivalue[0] = 2;
writeInt32Attribute(dataset, "NDArrayNumDims", 1, ivalue);
H5Gclose(groupDetector);
dims[0] = 1;
dims[1] = 1;
chunk[0] = 1;
rank = 1;
/* Unique ID */
datatype = H5T_NATIVE_INT32;
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_value(cparm, datatype, &fillValue);
H5Pset_chunk(cparm, rank, chunk);
dataspace = H5Screate_simple(rank, dims, maxdims);
dataset = H5Dcreate2(fid, "NDArrayUniqueId",
datatype, dataspace,
H5P_DEFAULT, cparm, H5P_DEFAULT);
memspace = H5Screate_simple(rank, elementSize, NULL);
writeStringAttribute(dataset, "NDAttrName", "NDArrayUniqueId");
writeStringAttribute(dataset, "NDAttrDescription", "The unique ID of the NDArray");
writeStringAttribute(dataset, "NDAttrSourceType", "NDAttrSourceDriver");
writeStringAttribute(dataset, "NDAttrSource", "Driver");
/* EPICS Timestemp */
datatype = H5T_NATIVE_DOUBLE;
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_value(cparm, datatype, &fillValue);
H5Pset_chunk(cparm, rank, chunk);
dataspace = H5Screate_simple(rank, dims, maxdims);
dataset = H5Dcreate2(fid, "NDArrayTimeStamp",
datatype, dataspace,
H5P_DEFAULT, cparm, H5P_DEFAULT);
memspace = H5Screate_simple(rank, elementSize, NULL);
writeStringAttribute(dataset, "NDAttrName", "NDArrayTimeStamp");
writeStringAttribute(dataset, "NDAttrDescription", "The timestamp of the NDArray as float64");
writeStringAttribute(dataset, "NDAttrSourceType", "NDAttrSourceDriver");
writeStringAttribute(dataset, "NDAttrSource", "Driver");
/* EPICS TS sec */
datatype = H5T_NATIVE_UINT32;
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_value(cparm, datatype, &fillValue);
H5Pset_chunk(cparm, rank, chunk);
dataspace = H5Screate_simple(rank, dims, maxdims);
dataset = H5Dcreate2(fid, "NDArrayEpicsTSSec",
datatype, dataspace,
H5P_DEFAULT, cparm, H5P_DEFAULT);
memspace = H5Screate_simple(rank, elementSize, NULL);
writeStringAttribute(dataset, "NDAttrName", "NDArrayEpicsTSSec");
writeStringAttribute(dataset, "NDAttrDescription", "The NDArray EPICS timestamp seconds past epoch");
writeStringAttribute(dataset, "NDAttrSourceType", "NDAttrSourceDriver");
writeStringAttribute(dataset, "NDAttrSource", "Driver");
/* EPICS TS nsec */
datatype = H5T_NATIVE_UINT32;
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_value(cparm, datatype, &fillValue);
H5Pset_chunk(cparm, rank, chunk);
dataspace = H5Screate_simple(rank, dims, maxdims);
dataset = H5Dcreate2(fid, "NDArrayEpicsTSnSec",
datatype, dataspace,
H5P_DEFAULT, cparm, H5P_DEFAULT);
memspace = H5Screate_simple(rank, elementSize, NULL);
writeStringAttribute(dataset, "NDAttrName", "NDArrayEpicsTSnSec");
writeStringAttribute(dataset, "NDAttrDescription", "The NDArray EPICS timestamp nanoseconds");
writeStringAttribute(dataset, "NDAttrSourceType", "NDAttrSourceDriver");
writeStringAttribute(dataset, "NDAttrSource", "Driver");
/* Color mode */
datatype = H5T_NATIVE_INT32;
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_value(cparm, datatype, &fillValue);
H5Pset_chunk(cparm, rank, chunk);
dataspace = H5Screate_simple(rank, dims, maxdims);
dataset = H5Dcreate2(fid, "ColorMode",
datatype, dataspace,
H5P_DEFAULT, cparm, H5P_DEFAULT);
memspace = H5Screate_simple(rank, elementSize, NULL);
writeStringAttribute(dataset, "NDAttrName", "ColorMode");
writeStringAttribute(dataset, "NDAttrDescription", "Color mode");
writeStringAttribute(dataset, "NDAttrSourceType", "NDAttrSourceDriver");
writeStringAttribute(dataset, "NDAttrSource", "Driver");
/* Camera manufacturer */
datatype = H5Tcopy(H5T_C_S1);
H5Tset_size(datatype, 256);
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_value(cparm, datatype, &fillValue);
H5Pset_chunk(cparm, rank, chunk);
dataspace = H5Screate_simple(rank, dims, maxdims);
dataset = H5Dcreate2(fid, "CameraManufacturer",
datatype, dataspace,
H5P_DEFAULT, cparm, H5P_DEFAULT);
memspace = H5Screate_simple(rank, elementSize, NULL);
writeStringAttribute(dataset, "NDAttrName", "CameraManufacturer");
writeStringAttribute(dataset, "NDAttrDescription", "Camera manufacturer");
writeStringAttribute(dataset, "NDAttrSourceType", "NDAttrSourceParam");
writeStringAttribute(dataset, "NDAttrSource", "MANUFACTURER");
/* Performance data */
rank = 2;
dims[0] = 1;
dims[1] = 5;
chunk[1] = 5;
cparm = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(cparm, rank, chunk);
dataspace = H5Screate_simple(rank, dims, maxdims);
dataset = H5Dcreate2(fid, "timestamp",
H5T_NATIVE_DOUBLE, dataspace,
H5P_DEFAULT, cparm, H5P_DEFAULT);
if (!H5Iis_valid(dataset)) {
printf("Error writing performance dataset");
}
H5Sclose(dataspace);
#if H5_VERSION_GE(1,9,178)
H5Fstart_swmr_write(fid);
#endif
H5Fclose(fid);
return 0;
} /* end main */
/*
* Function: do_write
* Purpose: Write the required amount of data to the file.
* Return: SUCCESS or FAIL
* Programmer: Christian Chilan, April, 2008
* Modifications:
*/
static herr_t
do_write(results *res, file_descr *fd, parameters *parms, void *buffer)
{
int ret_code = SUCCESS;
char dname[64];
int i;
size_t u;
/* HDF5 variables */
herr_t hrc; /*HDF5 return code */
hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */
hsize_t h5chunk[MAX_DIMS]; /*dataset dim sizes */
hsize_t h5block[MAX_DIMS]; /*dataspace selection */
hsize_t h5stride[MAX_DIMS]; /*selection stride */
hsize_t h5start[MAX_DIMS]; /*selection start */
hsize_t h5maxdims[MAX_DIMS];
int rank; /*rank of dataset */
/* Prepare buffer for verifying data */
/* if (parms->verify)
memset(buffer,1,linear_buf_size); */
buf_p=(unsigned char *)buffer;
for(u = 0; u < linear_buf_size; u++)
buf_p[u] = u % 128;
rank = parms->rank;
for(i = 0; i < rank; i++)
h5offset[i] = offset[i] = 0;
/* I/O Access specific setup */
switch (parms->io_type) {
case POSIXIO:
/* determine lowest dimension for contiguous POSIX access */
cont_dim = rank;
for (i=rank-1; i>=0; i--) {
if (parms->buf_size[i]==parms->dset_size[i])
cont_dim = i;
else
break;
}
/* determine size of the contiguous POSIX access */
cont_size = (!cont_dim)? 1 : parms->buf_size[cont_dim-1];
for (i=cont_dim; i<rank; i++)
cont_size *= parms->buf_size[i];
break;
case HDF5: /* HDF5 setup */
for (i=0; i < rank; i++){
h5dims[i] = parms->dset_size[i];
h5start[i] = 0;
h5stride[i] = 1;
h5block[i] = 1;
h5count[i] = parms->buf_size[i];
h5chunk[i] = parms->chk_size[i];
h5maxdims[i] = H5S_UNLIMITED;
}
if (parms->h5_use_chunks && parms->h5_extendable) {
h5dset_space_id = H5Screate_simple(rank, h5count, h5maxdims);
VRFY((h5dset_space_id >= 0), "H5Screate_simple");
}
else {
h5dset_space_id = H5Screate_simple(rank, h5dims, NULL);
VRFY((h5dset_space_id >= 0), "H5Screate_simple");
}
hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET,
h5start, h5stride, h5count, h5block);
VRFY((hrc >= 0), "H5Sselect_hyperslab");
/* Create the memory dataspace that corresponds to the xfer buffer */
h5mem_space_id = H5Screate_simple(rank, h5count, NULL);
VRFY((h5mem_space_id >= 0), "H5Screate_simple");
/* Create the dataset transfer property list */
h5dxpl = H5Pcreate(H5P_DATASET_XFER);
if (h5dxpl < 0) {
fprintf(stderr, "HDF5 Property List Create failed\n");
GOTOERROR(FAIL);
}
break;
default:
HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type);
GOTOERROR(FAIL);
break;
} /* end switch */
/* create dataset */
switch (parms->io_type) {
case POSIXIO:
break;
case HDF5:
h5dcpl = H5Pcreate(H5P_DATASET_CREATE);
if (h5dcpl < 0) {
fprintf(stderr, "HDF5 Property List Create failed\n");
GOTOERROR(FAIL);
}
if(parms->h5_use_chunks) {
/* Set the chunk size to be the same as the buffer size */
hrc = H5Pset_chunk(h5dcpl, rank, h5chunk);
if (hrc < 0) {
fprintf(stderr, "HDF5 Property List Set failed\n");
GOTOERROR(FAIL);
} /* end if */
} /* end if */
sprintf(dname, "Dataset_%ld", (unsigned long)parms->num_bytes);
h5ds_id = H5Dcreate2(fd->h5fd, dname, ELMT_H5_TYPE,
h5dset_space_id, H5P_DEFAULT, h5dcpl, H5P_DEFAULT);
if (h5ds_id < 0) {
HDfprintf(stderr, "HDF5 Dataset Create failed\n");
GOTOERROR(FAIL);
}
hrc = H5Pclose(h5dcpl);
/* verifying the close of the dcpl */
if (hrc < 0) {
HDfprintf(stderr, "HDF5 Property List Close failed\n");
GOTOERROR(FAIL);
}
break;
default:
/* unknown request */
HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type);
GOTOERROR(FAIL);
break;
}
/* Start "raw data" write timer */
set_time(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART);
/* Perform write */
hrc = dset_write(rank-1, fd, parms, buffer);
if (hrc < 0) {
fprintf(stderr, "Error in dataset write\n");
GOTOERROR(FAIL);
}
/* Stop "raw data" write timer */
set_time(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP);
/* Calculate write time */
/* Close dataset. Only HDF5 needs to do an explicit close. */
if (parms->io_type == HDF5) {
hrc = H5Dclose(h5ds_id);
if (hrc < 0) {
fprintf(stderr, "HDF5 Dataset Close failed\n");
GOTOERROR(FAIL);
}
h5ds_id = -1;
} /* end if */
done:
/* release HDF5 objects */
if (h5dset_space_id != -1) {
hrc = H5Sclose(h5dset_space_id);
if (hrc < 0){
fprintf(stderr, "HDF5 Dataset Space Close failed\n");
ret_code = FAIL;
} else {
h5dset_space_id = -1;
}
}
if (h5mem_space_id != -1) {
hrc = H5Sclose(h5mem_space_id);
if (hrc < 0) {
fprintf(stderr, "HDF5 Memory Space Close failed\n");
ret_code = FAIL;
} else {
h5mem_space_id = -1;
}
}
if (h5dxpl != -1) {
hrc = H5Pclose(h5dxpl);
if (hrc < 0) {
fprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n");
ret_code = FAIL;
} else {
h5dxpl = -1;
}
}
return ret_code;
}
int PHDF5fileClass::WritePHDF5dataset(string grpname, string datasetname, double ***data, int nx, int ny, int nz){
/* -------------------------- */
/* Local variables and arrays */
/* -------------------------- */
string dname;
double *buffer;
hid_t const h5type = H5T_NATIVE_DOUBLE;
hid_t glob_dspace;
hid_t locl_dspace;
hid_t dataset_prop;
hid_t dataset;
hid_t dataspace;
hid_t dataset_xfer;
/* --------------------------------- */
/* Check that dimensions are correct */
/* --------------------------------- */
if (bparticles && grpname.c_str()=="Particles"){
cout << " WARNING(phdf5): Particle data is not going to be written as the 'bparticles' flag is currently turn to FALSE" << endl;
return (2);
}
for (int i=0; i<ndim; i++){
if (dim[i]%chdim[i]!=0){
cout << " ERROR(phdf5): Grid size is not a multiple of the chunk size in the " << i << " dimension." << endl;
cout << " Glob: " << dim[0] << " " << dim[1] << " " << dim[2] << endl;
cout << " Locl: " << chdim[0] << " " << chdim[1] << " " << chdim[2] << endl;
return 1;
}
}
/* ----------------------- */
/* Copy raw data to buffer */
/* ----------------------- */
if (nx!=chdim[0] || ny!=chdim[1] || nz!=chdim[2]){
cout << " ERROR(phdf5): data size is not equal to HDF5 chunk size " << endl;
return 1;
}
buffer = new double[nx*ny*nz];
int l = 0;
for (int i = 0; i < nx; i++)
for (int j = 0; j < ny; j++)
for (int k = 0; k < nz; k++)
buffer[l++] = data[i][j][k];
/* -------------------------------------------------------- */
/* 5- Set the stride, count and block values for each chunk */
/* And set the offset for each chunk */
/* -------------------------------------------------------- */
hsize_t *stride = new hsize_t[ndim];
hsize_t *count = new hsize_t[ndim];
hsize_t *block = new hsize_t[ndim];
hsize_t *offset = new hsize_t[ndim];
for (int i=0; i<ndim; i++){
stride[i] = 1;
count[i] = 1;
block[i] = chdim[i];
offset[i] = mpicoord[i]*chdim[i];
}
/* ---------------------------------- */
/* 6- Create data spaces for our data */
/* ---------------------------------- */
glob_dspace = H5Screate_simple(ndim, dim, NULL);
locl_dspace = H5Screate_simple(ndim, chdim, NULL);
/* --------------------------------------- */
/* 7- Create the dataset for the HDF5 file */
/* --------------------------------------- */
dataset_prop = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(dataset_prop, ndim, chdim);
dname = "/"+grpname+"/"+datasetname;
dataset = H5Dcreate2(file_id, dname.c_str(), h5type, glob_dspace, H5P_DEFAULT, dataset_prop, H5P_DEFAULT);
H5Pclose(dataset_prop);
dataspace = H5Dget_space(dataset);
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block);
/* --------------------------------- */
/* 8- Set the parallel transfer mode */
/* --------------------------------- */
dataset_xfer = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dataset_xfer, H5FD_MPIO_COLLECTIVE);
/* ---------------------------- */
/* 9- Write data to the dataset */
/* ---------------------------- */
H5Dwrite(dataset, h5type, locl_dspace, dataspace, dataset_xfer, buffer);
delete [] buffer;
/* ------------------------------------------------------ */
/* Close dataset related variables created with H5*create */
/* ------------------------------------------------------ */
H5Pclose(dataset_xfer);
H5Dclose(dataset);
H5Sclose(locl_dspace);
/* ---------------------------------------------------- */
/* Close the remaining variables created with H5*create */
/* ---------------------------------------------------- */
delete [] stride;
delete [] count;
delete [] block;
delete [] offset;
}
int main (void)
{
hid_t file;
hid_t grp;
hid_t dataset, dataspace;
hid_t plist;
herr_t status;
hsize_t dims[2];
hsize_t cdims[2];
int idx_f, idx_g;
/*
* Create a file.
*/
file = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/*
* Create a group in the file.
*/
grp = H5Gcreate(file, "/Data", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
/*
* Create dataset "Compressed Data" in the group using absolute
* name. Dataset creation property list is modified to use
* GZIP compression with the compression effort set to 6.
* Note that compression can be used only when dataset is chunked.
*/
dims[0] = 1000;
dims[1] = 20;
cdims[0] = 20;
cdims[1] = 20;
dataspace = H5Screate_simple(RANK, dims, NULL);
plist = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(plist, 2, cdims);
H5Pset_deflate( plist, 6);
dataset = H5Dcreate(file, "/Data/Compressed_Data", H5T_NATIVE_INT,
dataspace, H5P_DEFAULT, plist, H5P_DEFAULT);
/*
* Close the first dataset .
*/
H5Sclose(dataspace);
H5Dclose(dataset);
/*
* Create the second dataset.
*/
dims[0] = 500;
dims[1] = 20;
dataspace = H5Screate_simple(RANK, dims, NULL);
dataset = H5Dcreate(file, "/Data/Float_Data", H5T_NATIVE_FLOAT,
dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
/*
*Close the second dataset and file.
*/
H5Sclose(dataspace);
H5Dclose(dataset);
H5Pclose(plist);
H5Gclose(grp);
H5Fclose(file);
/*
* Now reopen the file and group in the file.
*/
file = H5Fopen(H5FILE_NAME, H5F_ACC_RDWR, H5P_DEFAULT);
grp = H5Gopen(file, "Data", H5P_DEFAULT);
/*
* Access "Compressed_Data" dataset in the group.
*/
dataset = H5Dopen(grp, "Compressed_Data", H5P_DEFAULT);
if( dataset < 0) printf(" Dataset 'Compressed-Data' is not found. \n");
printf("\"/Data/Compressed_Data\" dataset is open \n");
/*
* Close the dataset.
*/
status = H5Dclose(dataset);
/*
* Create hard link to the Data group.
*/
status = H5Lcreate_hard(file, "Data", H5L_SAME_LOC, "Data_new", H5P_DEFAULT, H5P_DEFAULT);
/*
* We can access "Compressed_Data" dataset using created
* hard link "Data_new".
*/
dataset = H5Dopen(file, "/Data_new/Compressed_Data", H5P_DEFAULT);
if( dataset < 0) printf(" Dataset is not found. \n");
printf("\"/Data_new/Compressed_Data\" dataset is open \n");
/*
* Close the dataset.
*/
status = H5Dclose(dataset);
/*
* Use iterator to see the names of the objects in the root group.
*/
idx_f = H5Literate(file, H5_INDEX_NAME, H5_ITER_INC, NULL, file_info, NULL);
/*
* Unlink name "Data" and use iterator to see the names
* of the objects in the file root direvtory.
*/
if(H5Ldelete(file, "Data", H5P_DEFAULT) < 0)
printf(" H5Ldelete failed \n");
else
printf("\"Data\" is unlinked \n");
idx_f = H5Literate(file, H5_INDEX_NAME, H5_ITER_INC, NULL, file_info, NULL);
/*
* Use iterator to see the names of the objects in the group
* /Data_new.
*/
idx_g = H5Literate_by_name(grp, "/Data_new", H5_INDEX_NAME, H5_ITER_INC, NULL, group_info, NULL, H5P_DEFAULT);
/*
* Close the file.
*/
H5Gclose(grp);
H5Fclose(file);
return 0;
}
int FTI_WriteHDF5Var(FTIT_dataset *FTI_DataVar)
{
int j;
hsize_t dimLength[32];
char str[FTI_BUFS];
int res;
hid_t dcpl;
for (j = 0; j < FTI_DataVar->rank; j++) {
dimLength[j] = FTI_DataVar->dimLength[j];
}
dcpl = H5Pcreate (H5P_DATASET_CREATE);
res = H5Pset_fletcher32 (dcpl);
res = H5Pset_chunk (dcpl, FTI_DataVar->rank, dimLength);
hid_t dataspace = H5Screate_simple( FTI_DataVar->rank, dimLength, NULL);
hid_t dataset = H5Dcreate2 ( FTI_DataVar->h5group->h5groupID, FTI_DataVar->name,FTI_DataVar->type->h5datatype, dataspace, H5P_DEFAULT, dcpl , H5P_DEFAULT);
// If my data are stored in the CPU side
// Just store the data to the file and return;
#ifdef GPUSUPPORT
if ( !FTI_DataVar->isDevicePtr ){
#endif
res = H5Dwrite(dataset,FTI_DataVar->type->h5datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, FTI_DataVar->ptr);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
res = H5Pclose (dcpl);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
res = H5Dclose(dataset);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
res = H5Sclose(dataspace);
if (res < 0) {
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
return FTI_SCES;
#ifdef GPUSUPPORT
}
// This code is only executed in the GPU case.
hsize_t *count = (hsize_t*) malloc (sizeof(hsize_t)*FTI_DataVar->rank);
hsize_t *offset= (hsize_t*) calloc (FTI_DataVar->rank,sizeof(hsize_t));
if ( !count|| !offset){
sprintf(str, "Could Not allocate count and offset regions");
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
hsize_t seperator;
hsize_t fetchBytes = FTI_getHostBuffSize();
fetchBytes = FTI_calculateCountDim(FTI_DataVar->eleSize, fetchBytes ,count, FTI_DataVar->rank, dimLength, &seperator);
sprintf(str,"GPU-Device Message: I Will Fetch %lld Bytes Per Stream Request", fetchBytes);
FTI_Print(str,FTI_DBUG);
FTIT_data_prefetch prefetcher;
prefetcher.fetchSize = fetchBytes;
prefetcher.totalBytesToFetch = FTI_DataVar->size;
prefetcher.isDevice = FTI_DataVar->isDevicePtr;
prefetcher.dptr = FTI_DataVar->devicePtr;
size_t bytesToWrite;
FTI_InitPrefetcher(&prefetcher);
unsigned char *basePtr = NULL;
if ( FTI_Try(FTI_getPrefetchedData(&prefetcher, &bytesToWrite, &basePtr), "Fetch next memory block from GPU to write to HDF5") != FTI_SCES){
return FTI_NSCS;
}
while( basePtr ){
res = FTI_WriteElements( dataspace, FTI_DataVar->type->h5datatype, dataset, count, offset, FTI_DataVar->rank , basePtr);
if (res != FTI_SCES ) {
free(offset);
free(count);
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
FTI_AdvanceOffset(seperator, offset,count, dimLength, FTI_DataVar->rank);
if ( FTI_Try(FTI_getPrefetchedData(&prefetcher, &bytesToWrite, &basePtr),
"Fetch next memory block from GPU to write to HDF5") != FTI_SCES){
return FTI_NSCS;
}
}
res = H5Dclose(dataset);
if (res < 0) {
free(offset);
free(count);
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
res = H5Sclose(dataspace);
if (res < 0) {
free(offset);
free(count);
sprintf(str, "Dataset #%d could not be written", FTI_DataVar->id);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
free(offset);
free(count);
return FTI_SCES;
#endif
}
/*****************************************************************************
This function generates attributes, groups, and datasets of many types.
Parameters:
fname: file_name.
ngrps: number of top level groups.
ndsets: number of datasets.
attrs: number of attributes.
nrow: number of rows in a dataset.
chunk: chunk size (single number).
vlen: max vlen size.
comp: use latest format.
latest: use gzip comnpression.
Return: Non-negative on success/Negative on failure
Programmer: Peter Cao <*****@*****.**>, Jan. 2013
****************************************************************************/
herr_t create_perf_test_file(const char *fname, int ngrps, int ndsets,
int nattrs, hsize_t nrows, hsize_t dim0, hsize_t chunk, int vlen,
int compressed, int latest)
{
int i, j, k;
hid_t fid, sid_null, sid_scalar, sid_1d, sid_2d, did, aid, sid_2, sid_large,
fapl=H5P_DEFAULT, dcpl=H5P_DEFAULT, gid1, gid2, cmp_tid, tid_str,
tid_enum, tid_array_f, tid_vlen_i, tid_vlen_s;
char name[32], tmp_name1[32], tmp_name2[32], tmp_name3[32];
hsize_t dims[1]={dim0}, dims2d[2]={dim0, (dim0/4+1)}, dims_array[1]={FIXED_LEN},
dim1[1]={2};
char *enum_names[4] = {"SOLID", "LIQUID", "GAS", "PLASMA"};
test_comp_t *buf_comp=NULL, *buf_comp_large=NULL;
int *buf_int=NULL;
float (*buf_float_a)[FIXED_LEN]=NULL;
double **buf_double2d=NULL;
hvl_t *buf_vlen_i=NULL;
char (*buf_str)[FIXED_LEN];
char **buf_vlen_s=NULL;
hobj_ref_t buf_ref[2];
hdset_reg_ref_t buf_reg_ref[2];
size_t offset, len;
herr_t status;
char *names[NTYPES] = { "int", "ulong", "float", "double", "fixed string",
"enum", "fixed float array", "vlen int array", "vlen strings"};
hid_t types[NTYPES] = { H5T_NATIVE_INT, H5T_NATIVE_UINT64, H5T_NATIVE_FLOAT,
H5T_NATIVE_DOUBLE, tid_str, tid_enum, tid_array_f, tid_vlen_i, tid_vlen_s};
hsize_t coords[4][2] = { {0, 1}, {3, 5}, {1, 0}, {2, 4}}, start=0, stride=1, count=1;
if (nrows < NROWS) nrows = NROWS;
if (ngrps<NGROUPS) ngrps=NGROUPS;
if (ndsets<NDSETS) ndsets=NDSETS;
if (nattrs<NATTRS) nattrs=NATTRS;
if (dim0<DIM0) dim0=DIM0;
if (chunk>dim0) chunk=dim0/4;
if (chunk<1) chunk = 1;
if (vlen<1) vlen = MAXVLEN;
/* create fixed string datatype */
types[4] = tid_str = H5Tcopy (H5T_C_S1);
H5Tset_size (tid_str, FIXED_LEN);
/* create enum datatype */
types[5] = tid_enum = H5Tenum_create(H5T_NATIVE_INT);
for (i = (int) SOLID; i <= (int) PLASMA; i++) {
phase_t val = (phase_t) i;
status = H5Tenum_insert (tid_enum, enum_names[i], &val);
}
/* create float array datatype */
types[6] = tid_array_f = H5Tarray_create (H5T_NATIVE_FLOAT, 1, dims_array);
/* create variable length integer datatypes */
types[7] = tid_vlen_i = H5Tvlen_create (H5T_NATIVE_INT);
/* create variable length string datatype */
types[8] = tid_vlen_s = H5Tcopy (H5T_C_S1);
H5Tset_size (tid_vlen_s, H5T_VARIABLE);
/* create compound datatypes */
cmp_tid = H5Tcreate (H5T_COMPOUND, sizeof (test_comp_t));
offset = 0;
for (i=0; i<NTYPES-2; i++) {
H5Tinsert(cmp_tid, names[i], offset, types[i]);
offset += H5Tget_size(types[i]);
}
H5Tinsert(cmp_tid, names[7], offset, types[7]);
offset += sizeof (hvl_t);
H5Tinsert(cmp_tid, names[8], offset, types[8]);
/* create dataspace */
sid_1d = H5Screate_simple (1, dims, NULL);
sid_2d = H5Screate_simple (2, dims2d, NULL);
sid_2 = H5Screate_simple (1, dim1, NULL);
sid_large = H5Screate_simple (1, &nrows, NULL);
sid_null = H5Screate (H5S_NULL);
sid_scalar = H5Screate (H5S_SCALAR);
/* create fid access property */
fapl = H5Pcreate (H5P_FILE_ACCESS);
H5Pset_libver_bounds (fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
/* create dataset creation property */
dcpl = H5Pcreate (H5P_DATASET_CREATE);
/* set dataset chunk */
if (chunk>0) {
H5Pset_chunk (dcpl, 1, &chunk);
}
/* set dataset compression */
if (compressed) {
if (chunk<=0) {
chunk = dim0/10+1;;
H5Pset_chunk (dcpl, 1, &chunk);
}
H5Pset_shuffle (dcpl);
H5Pset_deflate (dcpl, 6);
}
/* allocate buffers */
buf_comp = (test_comp_t *)calloc(dim0, sizeof(test_comp_t));
buf_comp_large = (test_comp_t *)calloc(nrows, sizeof(test_comp_t));
buf_int = (int *)calloc(dim0, sizeof(int));
buf_float_a = malloc(dim0*sizeof(*buf_float_a));
buf_vlen_i = (hvl_t *)calloc(dim0, sizeof (hvl_t));
buf_vlen_s = (char **)calloc(dim0, sizeof(char *));
buf_str = malloc(dim0*sizeof (*buf_str));
/* allocate array of doulbe pointers */
buf_double2d = (double **)calloc(dims2d[0],sizeof(double *));
/* allocate a contigous chunk of memory for the data */
buf_double2d[0] = (double *)calloc( dims2d[0]*dims2d[1],sizeof(double) );
/* assign memory city to pointer array */
for (i=1; i <dims2d[0]; i++) buf_double2d[i] = buf_double2d[0]+i*dims2d[1];
/* fill buffer values */
len = 1;
for (i=0; i<dims[0]; i++) {
buf_comp[i].i = buf_int[i] = i-2147483648;
buf_comp[i].l = 0xffffffffffffffff-i;
buf_comp[i].f = 1.0/(i+1.0);
buf_comp[i].d = 987654321.0*i+1.0/(i+1.0);
buf_comp[i].e = (phase_t) (i % (int) (PLASMA + 1));
for (j=0; j<FIXED_LEN; j++) {
buf_comp[i].f_array[j] = buf_float_a[i][j] = i*100+j;
buf_str[i][j] = 'a' + (i%26);
}
buf_str[i][FIXED_LEN-1] = 0;
strcpy(buf_comp[i].s, buf_str[i]);
len = (1-cos(i/8.0))/2*vlen+1;
if (!i) len = vlen;
buf_vlen_i[i].len = len;
buf_vlen_i[i].p = (int *)calloc(len, sizeof(int));
for (j=0; j<len; j++) ((int*)(buf_vlen_i[i].p))[j] = i*100+j;
buf_comp[i].i_vlen = buf_vlen_i[i];
buf_vlen_s[i] = (char *)calloc(len, sizeof(char));
for (j=0; j<len-1; j++)
buf_vlen_s[i][j] = j%26+'A';
buf_comp[i].s_vlen = buf_vlen_s[i];
for (j=0; j<dims2d[1]; j++)
buf_double2d[i][j] = i+j/10000.0;
}
for (i=0; i<nrows; i++) {
buf_comp_large[i].i = i-2147483648;
buf_comp_large[i].l = 0xffffffffffffffff-i;
buf_comp_large[i].f = 1.0/(i+1.0);
buf_comp_large[i].d = 987654321.0*i+1.0/(i+1.0);
buf_comp_large[i].e = (phase_t) (i % (int) (PLASMA + 1));
for (j=0; j<FIXED_LEN-1; j++) {
buf_comp_large[i].f_array[j] = i*100+j;
buf_comp_large[i].s[j] = 'a' + (i%26);
}
len = i%vlen+1;
buf_comp_large[i].i_vlen.len = len;
buf_comp_large[i].i_vlen.p = (int *)calloc(len, sizeof(int));
for (j=0; j<len; j++) ((int*)(buf_comp_large[i].i_vlen.p))[j] = i*100+j;
buf_comp_large[i].s_vlen = (char *)calloc(i+2, sizeof(char));
for (j=0; j<i+1; j++) (buf_comp_large[i].s_vlen)[j] = j%26+'A';
}
/* create file */
if (latest)
fid = H5Fcreate (fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
else
fid = H5Fcreate (fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
add_attrs(fid, 0);
sprintf(name, "a cmp ds of %d rows", nrows);
did = H5Dcreate (fid, name, cmp_tid, sid_large, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, cmp_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_comp_large);
add_attrs(did, 0);
H5Dclose(did);
// /* add attributes*/
gid1 = H5Gcreate (fid, "attributes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (nattrs<1) nattrs = 1;
i=0;
while (i<nattrs) i += add_attrs(gid1, i);
H5Gclose(gid1);
/* add many sub groups to a group*/
gid1 = H5Gcreate (fid, "groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
add_attrs(gid1, 0);
for (i=0; i<ngrps; i++) {
/* create sub groups */
sprintf(name, "g%02d", i);
gid2 = H5Gcreate (gid1, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (i<10) add_attrs(gid2, 0);
H5Gclose(gid2);
}
H5Gclose(gid1);
/* add many datasets to a group */
gid1 = H5Gcreate (fid, "datasets", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
add_attrs(gid1, 0);
for (j=0; j<ndsets; j+=12) {
/* 1 add a null dataset */
sprintf(name, "%05d null dataset", j);
did = H5Dcreate (gid1, name, H5T_STD_I32LE, sid_null, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 2 add scalar int point */
sprintf(name, "%05d scalar int point", j);
did = H5Dcreate (gid1, name, H5T_NATIVE_INT, sid_scalar, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &j);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 3 scalar vlen string */
sprintf(name, "%05d scalar vlen string", j);
did = H5Dcreate (gid1, name, tid_vlen_s, sid_scalar, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, tid_vlen_s, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buf_vlen_s[0]);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 4 add fixed-length float array */
sprintf(name, "%05d fixed-length float array", j);
did = H5Dcreate (gid1, name, tid_array_f, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_array_f, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_float_a);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 5 add fixed-length strings */
sprintf(name, "%05d fixed-length strings", j);
did = H5Dcreate (gid1, name, tid_str, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_str, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_str);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 6 add compound data */
sprintf(name, "%05d compund data", j);
did = H5Dcreate (gid1, name, cmp_tid, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, cmp_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_comp);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 7 add 2D double */
sprintf(name, "%05d 2D double", j);
strcpy (tmp_name1, name);
did = H5Dcreate (gid1, name, H5T_NATIVE_DOUBLE, sid_2d, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_double2d[0]);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 8 add 1D int array */
sprintf(name, "%05d 1D int array", j);
did = H5Dcreate (gid1, name, H5T_NATIVE_INT, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_int);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 9 add vlen int array */
sprintf(name, "%05d vlen int array", j);
strcpy (tmp_name2, name);
did = H5Dcreate (gid1, name, tid_vlen_i, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_vlen_i, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_vlen_i);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 10 add vlen strings */
sprintf(name, "%05d vlen strings", j);
strcpy (tmp_name3, name);
did = H5Dcreate (gid1, name, tid_vlen_s, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_vlen_s, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_vlen_s);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 11 add object refs */
H5Rcreate(&buf_ref[0],gid1, ".", H5R_OBJECT, -1);
H5Rcreate(&buf_ref[1],gid1, tmp_name3, H5R_OBJECT, -1);
sprintf(name, "%05d obj refs", j);
did = H5Dcreate (gid1, name, H5T_STD_REF_OBJ, sid_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_ref);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 12 add region refs */
H5Sselect_elements (sid_2d, H5S_SELECT_SET, 4, coords[0]);
H5Rcreate(&buf_reg_ref[0],gid1, tmp_name1, H5R_DATASET_REGION, sid_2d);
H5Sselect_none(sid_2d);
count = dims[0]/2+1;
H5Sselect_hyperslab (sid_1d, H5S_SELECT_SET, &start, &stride, &count,NULL);
H5Rcreate(&buf_reg_ref[1],gid1, tmp_name2, H5R_DATASET_REGION, sid_1d);
H5Sselect_none(sid_1d);
sprintf(name, "%05d region refs", j);
did = H5Dcreate (gid1, name, H5T_STD_REF_DSETREG, sid_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_reg_ref);
if (!j) add_attrs(did, j);
H5Dclose(did);
}
H5Gclose(gid1);
H5Tclose (tid_array_f);
H5Tclose (tid_vlen_i);
H5Tclose (tid_vlen_s);
H5Tclose (tid_enum);
H5Tclose (tid_str);
H5Tclose (cmp_tid);
H5Pclose (dcpl);
H5Pclose (fapl);
H5Sclose (sid_1d);
H5Sclose (sid_2d);
H5Sclose (sid_2);
H5Sclose (sid_large);
H5Sclose (sid_null);
H5Sclose (sid_scalar);
H5Fclose (fid);
for (i=0; i<dims[0]; i++) {
if (buf_vlen_i[i].p) free(buf_vlen_i[i].p);
if (buf_vlen_s[i]) free(buf_vlen_s[i]);
}
for (i=0; i<nrows; i++) {
if (buf_comp_large[i].i_vlen.p) free(buf_comp_large[i].i_vlen.p);
if (buf_comp_large[i].s_vlen) free(buf_comp_large[i].s_vlen);
}
free (buf_comp);
free (buf_comp_large);
free (buf_int);
free (buf_float_a);
free (buf_double2d[0]);
free (buf_double2d);
free (buf_str);
free(buf_vlen_i);
free(buf_vlen_s);
return 0;
}
void _io_write_prim_h5mpi(const char *fname, const char **pnames, const double *data)
// -----------------------------------------------------------------------------
// This function uses a collective MPI-IO procedure to write the contents of
// 'data' to the HDF5 file named 'fname', which is assumed to have been created
// already. The dataset with name 'dname', which is being written to, must not
// exist already. Chunking is enabled as per the module-wide ChunkSize variable,
// and is disabled by default. Recommended chunk size is local subdomain
// size. This will result in optimized read/write on the same decomposition
// layout, but poor performance for different access patterns, for example the
// slabs used by cluster-FFT functions.
//
// WARNING!
//
// All processors must define the same chunk size, the behavior of this function
// is not defined otherwise. This implies that chunking should be disabled when
// running on a strange number of cores, and subdomain sizes are non-uniform.
// -----------------------------------------------------------------------------
{
hsize_t ndp1 = n_dims + 1;
hsize_t *a_nint = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
hsize_t *l_ntot = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
hsize_t *l_strt = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
hsize_t *stride = (hsize_t*) malloc(ndp1*sizeof(hsize_t));
int i;
for (i=0; i<n_dims; ++i) {
a_nint[i] = A_nint[i]; // Selection size, target and destination
l_ntot[i] = L_ntot[i]; // Memory space total size
l_strt[i] = L_strt[i]; // Memory space selection start
stride[i] = 1;
}
a_nint[ndp1 - 1] = 1;
l_ntot[ndp1 - 1] = n_prim;
stride[ndp1 - 1] = n_prim;
// Here we create the following property lists:
//
// file access property list ........ for the call to H5Fopen
// dset creation property list ........ for the call to H5Dcreate
// dset transfer property list ........ for the call to H5Dwrite
// ---------------------------------------------------------------------------
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
hid_t dxpl = H5Pcreate(H5P_DATASET_XFER);
// Here we define collective (MPI) access to the file with alignment
// properties optimized for the local file system, according to DiskBlockSize.
// ---------------------------------------------------------------------------
if (EnableChunking) {
H5Pset_chunk(dcpl, n_dims, ChunkSize);
}
if (EnableAlignment) {
H5Pset_alignment(fapl, AlignThreshold, DiskBlockSize);
}
H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
hid_t file = H5Fopen(fname, H5F_ACC_RDWR, fapl);
const int overwrite = H5Lexists(file, "prim", H5P_DEFAULT);
hid_t prim = overwrite ? H5Gopen(file, "prim", H5P_DEFAULT) :
H5Gcreate(file, "prim", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
hid_t mspc = H5Screate_simple(ndp1 , l_ntot, NULL);
hid_t fspc = H5Screate_simple(n_dims, G_ntot, NULL);
// Call signature to H5Sselect_hyperslab is (start, stride, count, chunk)
// ---------------------------------------------------------------------------
const clock_t start_all = clock();
for (i=0; i<n_prim; ++i) {
hid_t dset = overwrite ? H5Dopen(prim, pnames[i], H5P_DEFAULT) :
H5Dcreate(prim, pnames[i], H5T_NATIVE_DOUBLE, fspc,
H5P_DEFAULT, dcpl, H5P_DEFAULT);
l_strt[ndp1 - 1] = i;
H5Sselect_hyperslab(mspc, H5S_SELECT_SET, l_strt, stride, a_nint, NULL);
H5Sselect_hyperslab(fspc, H5S_SELECT_SET, G_strt, NULL, A_nint, NULL);
H5Dwrite(dset, H5T_NATIVE_DOUBLE, mspc, fspc, dxpl, data);
H5Dclose(dset);
}
if (iolog) {
const double sec = (double)(clock() - start_all) / CLOCKS_PER_SEC;
fprintf(iolog, "[h5mpi] write to %s took %f minutes\n", fname, sec/60.0);
fflush(iolog);
}
free(a_nint);
free(l_ntot);
free(l_strt);
// Always close the hid_t handles in the reverse order they were opened in.
// ---------------------------------------------------------------------------
H5Sclose(fspc);
H5Sclose(mspc);
H5Gclose(prim);
H5Fclose(file);
H5Pclose(dxpl);
H5Pclose(dcpl);
H5Pclose(fapl);
}
PetscErrorCode ISView_General_HDF5(IS is, PetscViewer viewer)
{
hid_t filespace; /* file dataspace identifier */
hid_t chunkspace; /* chunk dataset property identifier */
hid_t plist_id; /* property list identifier */
hid_t dset_id; /* dataset identifier */
hid_t memspace; /* memory dataspace identifier */
hid_t inttype; /* int type (H5T_NATIVE_INT or H5T_NATIVE_LLONG) */
hid_t file_id, group;
herr_t status;
hsize_t dim, maxDims[3], dims[3], chunkDims[3], count[3],offset[3];
PetscInt bs, N, n, timestep, low;
const PetscInt *ind;
const char *isname;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = ISGetBlockSize(is,&bs);CHKERRQ(ierr);
ierr = PetscViewerHDF5OpenGroup(viewer, &file_id, &group);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetTimestep(viewer, ×tep);CHKERRQ(ierr);
/* Create the dataspace for the dataset.
*
* dims - holds the current dimensions of the dataset
*
* maxDims - holds the maximum dimensions of the dataset (unlimited
* for the number of time steps with the current dimensions for the
* other dimensions; so only additional time steps can be added).
*
* chunkDims - holds the size of a single time step (required to
* permit extending dataset).
*/
dim = 0;
if (timestep >= 0) {
dims[dim] = timestep+1;
maxDims[dim] = H5S_UNLIMITED;
chunkDims[dim] = 1;
++dim;
}
ierr = ISGetSize(is, &N);CHKERRQ(ierr);
ierr = ISGetLocalSize(is, &n);CHKERRQ(ierr);
ierr = PetscHDF5IntCast(N/bs,dims + dim);CHKERRQ(ierr);
maxDims[dim] = dims[dim];
chunkDims[dim] = dims[dim];
++dim;
if (bs >= 1) {
dims[dim] = bs;
maxDims[dim] = dims[dim];
chunkDims[dim] = dims[dim];
++dim;
}
filespace = H5Screate_simple(dim, dims, maxDims);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Cannot H5Screate_simple()");
#if defined(PETSC_USE_64BIT_INDICES)
inttype = H5T_NATIVE_LLONG;
#else
inttype = H5T_NATIVE_INT;
#endif
/* Create the dataset with default properties and close filespace */
ierr = PetscObjectGetName((PetscObject) is, &isname);CHKERRQ(ierr);
if (!H5Lexists(group, isname, H5P_DEFAULT)) {
/* Create chunk */
chunkspace = H5Pcreate(H5P_DATASET_CREATE);
if (chunkspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Pcreate()");
status = H5Pset_chunk(chunkspace, dim, chunkDims);CHKERRQ(status);
#if (H5_VERS_MAJOR * 10000 + H5_VERS_MINOR * 100 + H5_VERS_RELEASE >= 10800)
dset_id = H5Dcreate2(group, isname, inttype, filespace, H5P_DEFAULT, chunkspace, H5P_DEFAULT);
#else
dset_id = H5Dcreate(group, isname, inttype, filespace, H5P_DEFAULT);
#endif
if (dset_id == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Dcreate2()");
status = H5Pclose(chunkspace);CHKERRQ(status);
} else {
dset_id = H5Dopen2(group, isname, H5P_DEFAULT);
status = H5Dset_extent(dset_id, dims);CHKERRQ(status);
}
status = H5Sclose(filespace);CHKERRQ(status);
/* Each process defines a dataset and writes it to the hyperslab in the file */
dim = 0;
if (timestep >= 0) {
count[dim] = 1;
++dim;
}
ierr = PetscHDF5IntCast(n/bs,count + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
count[dim] = bs;
++dim;
}
if (n > 0) {
memspace = H5Screate_simple(dim, count, NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate_simple()");
} else {
/* Can't create dataspace with zero for any dimension, so create null dataspace. */
memspace = H5Screate(H5S_NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate()");
}
/* Select hyperslab in the file */
ierr = PetscLayoutGetRange(is->map, &low, NULL);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
offset[dim] = timestep;
++dim;
}
ierr = PetscHDF5IntCast(low/bs,offset + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
offset[dim] = 0;
++dim;
}
if (n > 0) {
filespace = H5Dget_space(dset_id);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Dget_space()");
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL);CHKERRQ(status);
} else {
/* Create null filespace to match null memspace. */
filespace = H5Screate(H5S_NULL);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate(H5S_NULL)");
}
/* Create property list for collective dataset write */
plist_id = H5Pcreate(H5P_DATASET_XFER);
if (plist_id == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Pcreate()");
#if defined(PETSC_HAVE_H5PSET_FAPL_MPIO)
status = H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);CHKERRQ(status);
#endif
/* To write dataset independently use H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT) */
ierr = ISGetIndices(is, &ind);CHKERRQ(ierr);
status = H5Dwrite(dset_id, inttype, memspace, filespace, plist_id, ind);CHKERRQ(status);
status = H5Fflush(file_id, H5F_SCOPE_GLOBAL);CHKERRQ(status);
ierr = ISGetIndices(is, &ind);CHKERRQ(ierr);
/* Close/release resources */
if (group != file_id) {status = H5Gclose(group);CHKERRQ(status);}
status = H5Pclose(plist_id);CHKERRQ(status);
status = H5Sclose(filespace);CHKERRQ(status);
status = H5Sclose(memspace);CHKERRQ(status);
status = H5Dclose(dset_id);CHKERRQ(status);
ierr = PetscInfo1(is, "Wrote IS object with name %s\n", isname);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int
main(int argc, char **argv)
{
printf("\n*** Testing HDF5/NetCDF-4 interoperability...\n");
printf("*** testing HDF5 compatibility...");
{
#define GRPA_NAME "grpa"
#define VAR_NAME "vara"
#define NDIMS 2
int nrowCur = 7; /* current size */
int ncolCur = 3;
int nrowMax = nrowCur + 0; /* maximum size */
int ncolMax = ncolCur + 0;
hid_t xdimId;
hid_t ydimId;
hsize_t xscaleDims[1];
hsize_t yscaleDims[1];
hid_t xdimSpaceId, spaceId;
hid_t fileId;
hid_t fapl;
hsize_t curDims[2];
hsize_t maxDims[2];
hid_t dataTypeId, dsPropertyId, grpaId, grpaPropId, dsId;
hid_t ydimSpaceId;
const char * dimNameBase
= "This is a netCDF dimension but not a netCDF variable.";
char dimNameBuf[1000];
char *varaName = "/grpa/vara";
short amat[nrowCur][ncolCur];
int ii, jj;
xscaleDims[0] = nrowCur;
yscaleDims[0] = ncolCur;
if ((xdimSpaceId = H5Screate_simple(1, xscaleDims, NULL)) < 0) ERR;
/* With the SEMI close degree, the HDF5 file close will fail if
* anything is left open. */
if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR;
if (H5Pset_fclose_degree(fapl, H5F_CLOSE_SEMI)) ERR;
/* Create file */
if((fileId = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC,
H5Pcreate(H5P_FILE_CREATE), fapl)) < 0) ERR;
if (H5Pclose(fapl) < 0) ERR;
/* Create data space */
curDims[0] = nrowCur;
curDims[1] = ncolCur;
maxDims[0] = nrowMax;
maxDims[1] = ncolMax;
if ((spaceId = H5Screate_simple(2, curDims, maxDims)) < 0) ERR;
if ((dataTypeId = H5Tcopy(H5T_NATIVE_SHORT)) < 0) ERR;
if ((dsPropertyId = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
if ((grpaPropId = H5Pcreate(H5P_GROUP_CREATE)) < 0) ERR;
if ((grpaId = H5Gcreate2(fileId, GRPA_NAME, H5P_DEFAULT,
grpaPropId, H5P_DEFAULT)) < 0) ERR;
if (H5Pclose(grpaPropId) < 0) ERR;
/* Create vara dataset */
if ((dsId = H5Dcreate2(fileId, varaName, dataTypeId, spaceId,
H5P_DEFAULT, dsPropertyId,
H5P_DEFAULT)) < 0) ERR;
if (H5Pclose(dsPropertyId) < 0) ERR;
if (H5Tclose(dataTypeId) < 0) ERR;
if ((ydimSpaceId = H5Screate_simple(1, yscaleDims, NULL)) < 0) ERR;
/* Create xdim dimension dataset */
if ((xdimId = H5Dcreate2(fileId, "/xdim", H5T_IEEE_F32BE,
xdimSpaceId, H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT)) < 0) ERR;
if (H5Sclose(xdimSpaceId) < 0) ERR;
/* Create ydim dimension dataset */
if ((ydimId = H5Dcreate2(fileId, "/ydim", H5T_IEEE_F32BE,
ydimSpaceId, H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT)) < 0) ERR;
if (H5Sclose(ydimSpaceId) < 0) ERR;
/* Create xdim scale */
sprintf(dimNameBuf, "%s%10d", dimNameBase, nrowCur);
if (H5DSset_scale(xdimId, dimNameBuf) < 0) ERR;
/* Create ydim scale */
sprintf(dimNameBuf, "%s%10d", dimNameBase, ncolCur);
if (H5DSset_scale(ydimId, dimNameBuf) < 0) ERR;
/* Attach dimension scales to the dataset */
if (H5DSattach_scale(dsId, xdimId, 0) < 0) ERR;
if (H5DSattach_scale(dsId, ydimId, 1) < 0) ERR;
/* Close stuff. */
if (H5Dclose(xdimId) < 0) ERR;
if (H5Dclose(ydimId) < 0) ERR;
if (H5Dclose(dsId) < 0) ERR;
if (H5Gclose(grpaId) < 0) ERR;
if (H5Sclose(spaceId) < 0) ERR;
if (H5Fclose(fileId) < 0) ERR;
/* Create some data */
for (ii = 0; ii < nrowCur; ii++)
for (jj = 0; jj < ncolCur; jj++)
amat[ii][jj] = 100 * ii + jj;
/* Re-open file */
if ((fileId = H5Fopen(FILE_NAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) ERR;
if ((grpaId = H5Gopen2(fileId, GRPA_NAME, H5P_DEFAULT)) < 0) ERR;
if ((dsId = H5Dopen2(grpaId, varaName, H5P_DEFAULT)) < 0) ERR;
/* Write dataset */
if (H5Dwrite(dsId, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
amat) < 0) ERR;
/* Write dimension values for both xdim, ydim */
{
short xydimMat[ nrowCur >= ncolCur ? nrowCur : ncolCur];
for (ii = 0; ii < nrowCur; ii++)
xydimMat[ii] = 0; /*#### 100 * ii; */
/* Write xdim */
if ((xdimId = H5Dopen2(fileId, "/xdim", H5P_DEFAULT)) < 0) ERR;
if (H5Dwrite(xdimId, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, xydimMat) < 0) ERR;
if (H5Dclose(xdimId) < 0) ERR;
/* Write ydim */
if ((ydimId = H5Dopen2(fileId, "/ydim", H5P_DEFAULT)) < 0) ERR;
if (H5Dwrite(ydimId, H5T_NATIVE_SHORT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, xydimMat) < 0) ERR;
if (H5Dclose(ydimId) < 0) ERR;
}
if (H5Dclose(dsId) < 0) ERR;
if (H5Gclose(grpaId) < 0) ERR;
if (H5Fclose(fileId) < 0) ERR;
{
int ncid, grpid, nvars, ngatts, ndims, unlimdimid, ngrps;
char name_in[NC_MAX_NAME + 1];
nc_type xtype_in;
int ndims_in, natts_in, dimid_in[NDIMS];
/* nc_set_log_level(5);*/
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (ndims != 2 || nvars != 0 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_grps(ncid, &ngrps, &grpid)) ERR;
if (ngrps != 1) ERR;
if (nc_inq(grpid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (ndims != 0 || nvars != 1 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_var(grpid, 0, name_in, &xtype_in, &ndims_in, dimid_in,
&natts_in)) ERR;
if (strcmp(name_in, VAR_NAME) || xtype_in != NC_SHORT || ndims_in != NDIMS ||
dimid_in[0] != 0 || dimid_in[1] != 1 || natts_in != 0) ERR;
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
#ifdef USE_SZIP
printf("*** testing HDF5 compatibility with szip...");
{
#define DEFLATE_LEVEL 9
#define MAX_NAME 100
#define NUM_CD_ELEM 10
/* HDF5 defines this... */
#define DEFLATE_NAME "deflate"
#define DIM1_LEN 3000
#define GRP_NAME "George_Washington"
#define BATTLE_RECORD "Battle_Record"
hid_t fileid, grpid, spaceid, datasetid;
int data_out[DIM1_LEN], data_in[DIM1_LEN];
hsize_t dims[1] = {DIM1_LEN};
hid_t propid;
char name_in[MAX_NAME + 1];
int ncid, ndims_in, nvars_in, ngatts_in, unlimdimid_in, ngrps_in;
int nc_grpid;
int dimid_in[1], natts_in;
nc_type xtype_in;
int i;
for (i = 0; i < DIM1_LEN; i++)
data_out[i] = i;
/* Open file and create group. */
if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT,
H5P_DEFAULT)) < 0) ERR;
if ((grpid = H5Gcreate(fileid, GRP_NAME, 0)) < 0) ERR;
/* Write an array of bools, with szip compression. */
if ((propid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
if (H5Pset_layout(propid, H5D_CHUNKED)) ERR;
if (H5Pset_chunk(propid, 1, dims)) ERR;
if (H5Pset_szip(propid, H5_SZIP_EC_OPTION_MASK, 32)) ERR;
if ((spaceid = H5Screate_simple(1, dims, dims)) < 0) ERR;
if ((datasetid = H5Dcreate(grpid, BATTLE_RECORD, H5T_NATIVE_INT,
spaceid, propid)) < 0) ERR;
if (H5Dwrite(datasetid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
data_out) < 0) ERR;
if (H5Dclose(datasetid) < 0 ||
H5Pclose(propid) < 0 ||
H5Sclose(spaceid) < 0 ||
H5Gclose(grpid) < 0 ||
H5Fclose(fileid) < 0)
ERR;
/* Open the file with netCDF and check it. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in)) ERR;
if (ndims_in != 0 || nvars_in != 0 || ngatts_in != 0 || unlimdimid_in != -1) ERR;
if (nc_inq_grps(ncid, &ngrps_in, &nc_grpid)) ERR;
if (ngrps_in != 1) ERR;
if (nc_inq(nc_grpid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in)) ERR;
if (ndims_in != 1 || nvars_in != 1 || ngatts_in != 0 || unlimdimid_in != -1) ERR;
/* Check the variable. */
if (nc_inq_var(nc_grpid, 0, name_in, &xtype_in, &ndims_in, dimid_in,
&natts_in)) ERR;
if (strcmp(name_in, BATTLE_RECORD) || xtype_in != NC_INT || ndims_in != 1 ||
dimid_in[0] != 0 || natts_in != 0) ERR;
/* Check the data. */
if (nc_get_var(nc_grpid, 0, data_in)) ERR;
for (i = 0; i < DIM1_LEN; i++)
if (data_in[i] != data_out[i]) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#endif /* USE_SZIP */
FINAL_RESULTS;
}
int apply_filters(const char* name, /* object name from traverse list */
int rank, /* rank of dataset */
hsize_t *dims, /* dimensions of dataset */
size_t msize, /* size of type */
hid_t dcpl_id, /* dataset creation property list */
pack_opt_t *options, /* repack options */
int *has_filter) /* (OUT) object NAME has a filter */
{
int nfilters; /* number of filters in DCPL */
hsize_t chsize[64]; /* chunk size in elements */
H5D_layout_t layout;
int i;
pack_info_t obj;
*has_filter = 0;
if (rank==0) /* scalar dataset, do not apply */
return 0;
/*-------------------------------------------------------------------------
* initialize the assigment object
*-------------------------------------------------------------------------
*/
init_packobject(&obj);
/*-------------------------------------------------------------------------
* find options
*-------------------------------------------------------------------------
*/
if (aux_assign_obj(name,options,&obj)==0)
return 0;
/* get information about input filters */
if ((nfilters = H5Pget_nfilters(dcpl_id))<0)
return -1;
/*-------------------------------------------------------------------------
* check if we have filters in the pipeline
* we want to replace them with the input filters
* only remove if we are inserting new ones
*-------------------------------------------------------------------------
*/
if (nfilters && obj.nfilters )
{
*has_filter = 1;
if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0)
return -1;
}
/*-------------------------------------------------------------------------
* check if there is an existent chunk
* read it only if there is not a requested layout
*-------------------------------------------------------------------------
*/
if (obj.layout == -1 )
{
if ((layout = H5Pget_layout(dcpl_id))<0)
return -1;
if (layout == H5D_CHUNKED)
{
if ((rank = H5Pget_chunk(dcpl_id,NELMTS(chsize),chsize/*out*/))<0)
return -1;
obj.layout = H5D_CHUNKED;
obj.chunk.rank = rank;
for ( i = 0; i < rank; i++)
obj.chunk.chunk_lengths[i] = chsize[i];
}
}
/*-------------------------------------------------------------------------
* the type of filter and additional parameter
* type can be one of the filters
* H5Z_FILTER_NONE 0 , uncompress if compressed
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
* H5Z_FILTER_SZIP 4 , szip compression
* H5Z_FILTER_NBIT 5 , nbit compression
* H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression
*-------------------------------------------------------------------------
*/
if (obj.nfilters)
{
/*-------------------------------------------------------------------------
* filters require CHUNK layout; if we do not have one define a default
*-------------------------------------------------------------------------
*/
if (obj.layout==-1)
{
/* stripmine info */
hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */
hsize_t sm_nbytes; /*bytes per stripmine */
obj.chunk.rank = rank;
/*
* determine the strip mine size. The strip mine is
* a hyperslab whose size is manageable.
*/
sm_nbytes = msize;
for ( i = rank; i > 0; --i)
{
hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
size = 1;
sm_size[i - 1] = MIN(dims[i - 1], size);
sm_nbytes *= sm_size[i - 1];
assert(sm_nbytes > 0);
}
for ( i = 0; i < rank; i++)
{
obj.chunk.chunk_lengths[i] = sm_size[i];
}
}
for ( i=0; i<obj.nfilters; i++)
{
switch (obj.filter[i].filtn)
{
default:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_DEFLATE:
{
unsigned aggression; /* the deflate level */
aggression = obj.filter[i].cd_values[0];
/* set up for deflated data */
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if(H5Pset_deflate(dcpl_id,aggression)<0)
return -1;
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SZIP 4 , szip compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SZIP:
{
unsigned options_mask;
unsigned pixels_per_block;
options_mask = obj.filter[i].cd_values[0];
pixels_per_block = obj.filter[i].cd_values[1];
/* set up for szip data */
if(H5Pset_chunk(dcpl_id,obj.chunk.rank,obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_szip(dcpl_id,options_mask,pixels_per_block)<0)
return -1;
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SHUFFLE:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_shuffle(dcpl_id)<0)
return -1;
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_FLETCHER32:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_fletcher32(dcpl_id)<0)
return -1;
break;
/*----------- -------------------------------------------------------------
* H5Z_FILTER_NBIT , NBIT compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_NBIT:
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_nbit(dcpl_id)<0)
return -1;
break;
/*----------- -------------------------------------------------------------
* H5Z_FILTER_SCALEOFFSET , scale+offset compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SCALEOFFSET:
{
H5Z_SO_scale_type_t scale_type;
int scale_factor;
scale_type = (H5Z_SO_scale_type_t)obj.filter[i].cd_values[0];
scale_factor = obj.filter[i].cd_values[1];
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
if (H5Pset_scaleoffset(dcpl_id,scale_type,scale_factor)<0)
return -1;
}
break;
} /* switch */
}/*i*/
}
/*obj.nfilters*/
/*-------------------------------------------------------------------------
* layout
*-------------------------------------------------------------------------
*/
if (obj.layout>=0)
{
/* a layout was defined */
if (H5Pset_layout(dcpl_id, obj.layout)<0)
return -1;
if (H5D_CHUNKED == obj.layout)
{
if(H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths)<0)
return -1;
}
else if (H5D_COMPACT == obj.layout)
{
if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY)<0)
return -1;
}
/* remove filters for the H5D_CONTIGUOUS case */
else if (H5D_CONTIGUOUS == obj.layout)
{
if (H5Premove_filter(dcpl_id,H5Z_FILTER_ALL)<0)
return -1;
}
}
return 0;
}
/*-------------------------------------------------------------------------
* Function: create_szip_dsets_float
*
* Purpose: Create a dataset of FLOAT datatype with szip filter
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 29 March 2011
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
create_szip_dsets_float(hid_t fid, hid_t fsid, hid_t msid)
{
hid_t dataset; /* dataset handles */
hid_t dcpl;
float data[NX][NY]; /* data to write */
float fillvalue = -2.2f;
hsize_t chunk[RANK] = {CHUNK0, CHUNK1};
int i, j;
/*
* Data and output buffer initialization.
*/
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++)
data[j][i] = ((float)(i + j + 1))/3;
}
/*
* Create the dataset creation property list, add the Scale-Offset
* filter, set the chunk size, and set the fill value.
*/
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_szip(dcpl, H5_SZIP_NN_OPTION_MASK, 4) < 0)
TEST_ERROR
if(H5Pset_chunk(dcpl, RANK, chunk) < 0)
TEST_ERROR
if(H5Pset_fill_value(dcpl, H5T_NATIVE_FLOAT, &fillvalue) < 0)
TEST_ERROR
/*
* Create a new dataset within the file using defined dataspace, little
* endian datatype and default dataset creation properties.
*/
if((dataset = H5Dcreate2(fid, DATASETNAME18, H5T_IEEE_F32LE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
/*
* Write the data to the dataset using default transfer properties.
*/
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
/* Close dataset */
if(H5Dclose(dataset) < 0)
TEST_ERROR
/* Now create a dataset with a big-endian type */
if((dataset = H5Dcreate2(fid, DATASETNAME19, H5T_IEEE_F32BE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
if(H5Dclose(dataset) < 0)
TEST_ERROR
/*
* Close/release resources.
*/
if(H5Pclose(dcpl) < 0)
TEST_ERROR
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(dataset);
} H5E_END_TRY;
return -1;
}
/*-------------------------------------------------------------------------
* Function: create_deflate_dsets_float
*
* Purpose: Create a dataset of FLOAT datatype with deflate filter
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 29 March 2011
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
create_deflate_dsets_float(hid_t fid, hid_t fsid, hid_t msid)
{
#ifdef H5_HAVE_FILTER_DEFLATE
hid_t dataset = -1; /* dataset handles */
hid_t dcpl = -1;
float data[NX][NY]; /* data to write */
float fillvalue = -2.2f;
hsize_t chunk[RANK] = {CHUNK0, CHUNK1};
int i, j;
/*
* Data and output buffer initialization.
*/
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++)
data[j][i] = ((float)(i + j + 1))/3;
}
/*
* Create the dataset creation property list, add the Scale-Offset
* filter, set the chunk size, and set the fill value.
*/
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_deflate (dcpl, 6) < 0)
TEST_ERROR
if(H5Pset_chunk(dcpl, RANK, chunk) < 0)
TEST_ERROR
if(H5Pset_fill_value(dcpl, H5T_NATIVE_FLOAT, &fillvalue) < 0)
TEST_ERROR
/*
* Create a new dataset within the file using defined dataspace, little
* endian datatype and default dataset creation properties.
*/
if((dataset = H5Dcreate2(fid, DATASETNAME16, H5T_IEEE_F32LE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
/*
* Write the data to the dataset using default transfer properties.
*/
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
/* Close dataset */
if(H5Dclose(dataset) < 0)
TEST_ERROR
/* Now create a dataset with a big-endian type */
if((dataset = H5Dcreate2(fid, DATASETNAME17, H5T_IEEE_F32BE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
if(H5Dclose(dataset) < 0)
TEST_ERROR
/*
* Close/release resources.
*/
if(H5Pclose(dcpl) < 0)
TEST_ERROR
#else /* H5_HAVE_FILTER_DEFLATE */
const char *not_supported= "Deflate filter is not enabled. Can't create the dataset.";
puts(not_supported);
#endif /* H5_HAVE_FILTER_DEFLATE */
return 0;
#ifdef H5_HAVE_FILTER_DEFLATE
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(dataset);
} H5E_END_TRY;
return -1;
#endif /* H5_HAVE_FILTER_DEFLATE */
}
/*-------------------------------------------------------------------------
* Function: create_scale_offset_dset_long_long
*
* Purpose: Create a dataset of LONG LONG datatype with scale-offset
* filter
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Neil Fortner
* 27 January 2011
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
create_scale_offset_dsets_long_long(hid_t fid, hid_t fsid, hid_t msid)
{
hid_t dataset = -1; /* dataset handles */
hid_t dcpl = -1;
long long data[NX][NY]; /* data to write */
long long fillvalue = -2;
hsize_t chunk[RANK] = {CHUNK0, CHUNK1};
int i, j;
/*
* Data and output buffer initialization.
*/
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++)
data[j][i] = i + j;
}
/*
* 0 1 2 3 4 5
* 1 2 3 4 5 6
* 2 3 4 5 6 7
* 3 4 5 6 7 8
* 4 5 6 7 8 9
* 5 6 7 8 9 10
*/
/*
* Create the dataset creation property list, add the Scale-Offset
* filter, set the chunk size, and set the fill value.
*/
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_scaleoffset(dcpl, H5Z_SO_INT, H5Z_SO_INT_MINBITS_DEFAULT) < 0)
TEST_ERROR
if(H5Pset_chunk(dcpl, RANK, chunk) < 0)
TEST_ERROR
if(H5Pset_fill_value(dcpl, H5T_NATIVE_LLONG, &fillvalue) < 0)
TEST_ERROR
/*
* Create a new dataset within the file using defined dataspace, little
* endian datatype and default dataset creation properties.
*/
if((dataset = H5Dcreate2(fid, DATASETNAME12, H5T_STD_I64LE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
/*
* Write the data to the dataset using default transfer properties.
*/
if(H5Dwrite(dataset, H5T_NATIVE_LLONG, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
/* Close dataset */
if(H5Dclose(dataset) < 0)
TEST_ERROR
/* Now create a dataset with a big-endian type */
if((dataset = H5Dcreate2(fid, DATASETNAME13, H5T_STD_I64BE, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
if(H5Dwrite(dataset, H5T_NATIVE_LLONG, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
if(H5Dclose(dataset) < 0)
TEST_ERROR
/*
* Close/release resources.
*/
if(H5Pclose(dcpl) < 0)
TEST_ERROR
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(dataset);
} H5E_END_TRY;
return -1;
}
/*-------------------------------------------------------------------------
* Function: create_nbit_dsets_float
*
* Purpose: Create a dataset of FLOAT datatype with nbit filter
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 29 March 2011
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
create_nbit_dsets_float(hid_t fid, hid_t fsid, hid_t msid)
{
hid_t dataset = -1; /* dataset handles */
hid_t datatype = -1;
hid_t dcpl = -1;
size_t precision, offset;
float data[NX][NY]; /* data to write */
float fillvalue = -2.2f;
hsize_t chunk[RANK] = {CHUNK0, CHUNK1};
int i, j;
/*
* Data and output buffer initialization.
*/
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++)
data[j][i] = ((float)(i + j + 1))/3;
}
/*
* Create the dataset creation property list, add the Scale-Offset
* filter, set the chunk size, and set the fill value.
*/
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
if(H5Pset_nbit(dcpl) < 0)
TEST_ERROR
if(H5Pset_chunk(dcpl, RANK, chunk) < 0)
TEST_ERROR
if(H5Pset_fill_value(dcpl, H5T_NATIVE_FLOAT, &fillvalue) < 0)
TEST_ERROR
/* Define user-defined single-precision floating-point type for dataset.
* A 20-bit little-endian data type. */
if((datatype = H5Tcopy(H5T_IEEE_F32LE)) < 0)
TEST_ERROR
if(H5Tset_fields(datatype, (size_t)26, (size_t)20, (size_t)6, (size_t)7, (size_t)13) < 0)
TEST_ERROR
offset = 7;
if(H5Tset_offset(datatype,offset) < 0)
TEST_ERROR
precision = 20;
if(H5Tset_precision(datatype,precision) < 0)
TEST_ERROR
if(H5Tset_size(datatype, (size_t)4) < 0)
TEST_ERROR
if(H5Tset_ebias(datatype, (size_t)31) < 0)
TEST_ERROR
/*
* Create a new dataset within the file using defined dataspace,
* user-defined datatype, and default dataset creation properties.
*/
if((dataset = H5Dcreate2(fid, DATASETNAME22, datatype, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
/*
* Write the data to the dataset using default transfer properties.
*/
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
/* Close dataset */
if(H5Dclose(dataset) < 0)
TEST_ERROR
/* Now create a dataset with a big-endian type */
if(H5Tset_order(datatype, H5T_ORDER_BE) < 0)
TEST_ERROR
if((dataset = H5Dcreate2(fid, DATASETNAME23, datatype, fsid,
H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
if(H5Dwrite(dataset, H5T_NATIVE_FLOAT, msid, fsid, H5P_DEFAULT, data) < 0)
TEST_ERROR
if(H5Dclose(dataset) < 0)
TEST_ERROR
/*
* Close/release resources.
*/
if(H5Pclose(dcpl) < 0)
TEST_ERROR
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(dataset);
} H5E_END_TRY;
return -1;
}
/*
* This creates a dataset serially with 'nchunks' chunks, each of CHUNKSIZE
* elements. The allocation time is set to H5D_ALLOC_TIME_EARLY. Another
* routine will open this in parallel for extension test.
*/
void
create_chunked_dataset(const char *filename, int nchunks, write_type write_pattern)
{
hid_t file_id, dataset; /* handles */
hid_t dataspace,memspace;
hid_t cparms;
hsize_t dims[1];
hsize_t maxdims[1] = {H5S_UNLIMITED};
hsize_t chunk_dims[1] ={CHUNKSIZE};
hsize_t count[1];
hsize_t stride[1];
hsize_t block[1];
hsize_t offset[1]; /* Selection offset within dataspace */
/* Variables used in reading data back */
char buffer[CHUNKSIZE];
int i;
herr_t hrc;
MPI_Offset filesize, /* actual file size */
est_filesize; /* estimated file size */
/* set up MPI parameters */
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
/* Only MAINPROCESS should create the file. Others just wait. */
if (MAINPROCESS){
dims[0]=nchunks*CHUNKSIZE;
/* Create the data space with unlimited dimensions. */
dataspace = H5Screate_simple (1, dims, maxdims);
VRFY((dataspace >= 0), "");
memspace = H5Screate_simple(1, chunk_dims, NULL);
VRFY((memspace >= 0), "");
/* Create a new file. If file exists its contents will be overwritten. */
file_id = H5Fcreate(h5_rmprefix(filename), H5F_ACC_TRUNC, H5P_DEFAULT,
H5P_DEFAULT);
VRFY((file_id >= 0), "H5Fcreate");
/* Modify dataset creation properties, i.e. enable chunking */
cparms = H5Pcreate (H5P_DATASET_CREATE);
VRFY((cparms >= 0), "");
hrc = H5Pset_alloc_time(cparms, H5D_ALLOC_TIME_EARLY);
VRFY((hrc >= 0), "");
hrc = H5Pset_chunk ( cparms, 1, chunk_dims);
VRFY((hrc >= 0), "");
/* Create a new dataset within the file using cparms creation properties. */
dataset = H5Dcreate (file_id, DATASETNAME, H5T_NATIVE_UCHAR, dataspace, cparms);
VRFY((dataset >= 0), "");
switch (write_pattern) {
/* writes only the second to last chunk */
case sec_last:
memset(buffer, 100, CHUNKSIZE);
count[0] = 1;
stride[0] = 1;
block[0] = chunk_dims[0];
offset[0] = (nchunks-2)*chunk_dims[0];
hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block);
VRFY((hrc >= 0), "");
/* Write sec_last chunk */
hrc = H5Dwrite(dataset, H5T_NATIVE_UCHAR, memspace, dataspace, H5P_DEFAULT, buffer);
VRFY((hrc >= 0), "H5Dwrite");
break;
/* doesn't write anything */
case none:
break;
}
/* Close resources */
hrc = H5Dclose (dataset);
VRFY((hrc >= 0), "");
dataset = -1;
hrc = H5Sclose (dataspace);
VRFY((hrc >= 0), "");
hrc = H5Sclose (memspace);
VRFY((hrc >= 0), "");
hrc = H5Pclose (cparms);
VRFY((hrc >= 0), "");
hrc = H5Fclose (file_id);
VRFY((hrc >= 0), "");
file_id = -1;
/* verify file size */
filesize = get_filesize(filename);
est_filesize = nchunks*CHUNKSIZE*sizeof(unsigned char);
VRFY((filesize >= est_filesize), "file size check");
}
/* Make sure all processes are done before exiting this routine. Otherwise,
* other tests may start and change the test data file before some processes
* of this test are still accessing the file.
*/
MPI_Barrier(MPI_COMM_WORLD);
}
