H5Dataset::H5Layout & H5Dataset::getLayout() { hid_t plist = H5Dget_create_plist(dataset); H5D_layout_t layout = H5Pget_layout(plist); H5Dataset::H5Layout * obj = 0; switch (layout) { case H5D_COMPACT: obj = new H5Dataset::H5CompactLayout(*this); break; case H5D_CONTIGUOUS: obj = new H5Dataset::H5ContiguousLayout(*this); break; case H5D_CHUNKED: obj = new H5Dataset::H5ChunkedLayout(*this); break; default: H5Pclose(plist); throw H5Exception(__LINE__, __FILE__, _("Invalid layout")); } H5Pclose(plist); return *obj; }
herr_t H5ARRAYget_fill_value( hid_t dataset_id, hid_t type_id, int *status, void *value) { hid_t plist_id; /* Get creation properties list */ if ( (plist_id = H5Dget_create_plist(dataset_id)) < 0 ) goto out; /* How the fill value is defined? */ if ( (H5Pfill_value_defined(plist_id, status)) < 0 ) goto out; if ( *status == H5D_FILL_VALUE_USER_DEFINED ) { if ( H5Pget_fill_value(plist_id, type_id, value) < 0 ) goto out; } /* Terminate access to the datatype */ if ( H5Pclose( plist_id ) < 0 ) goto out; return 0; out: return -1; }
herr_t H5ARRAYget_chunkshape( hid_t dataset_id, int rank, hsize_t *dims_chunk) { hid_t plist_id; H5D_layout_t layout; /* Get creation properties list */ if ( (plist_id = H5Dget_create_plist( dataset_id )) < 0 ) goto out; /* Get the dataset layout */ layout = H5Pget_layout(plist_id); if (layout != H5D_CHUNKED) { H5Pclose( plist_id ); return -1; } /* Get the chunkshape for all dimensions */ if (H5Pget_chunk(plist_id, rank, dims_chunk ) < 0 ) goto out; /* Terminate access to the datatype */ if ( H5Pclose( plist_id ) < 0 ) goto out; return 0; out: if (dims_chunk) free(dims_chunk); return -1; }
/**************************************************************** ** ** getHDF5ClassID(): Returns class ID for loc_id.name. -1 if error. ** ****************************************************************/ H5T_class_t getHDF5ClassID(hid_t loc_id, const char *name, H5D_layout_t *layout, hid_t *type_id, hid_t *dataset_id) { H5T_class_t class_id; hid_t plist; /* Open the dataset. */ if ( (*dataset_id = H5Dopen( loc_id, name, H5P_DEFAULT )) < 0 ) return -1; /* Get an identifier for the datatype. */ *type_id = H5Dget_type( *dataset_id ); /* Get the class. */ class_id = H5Tget_class( *type_id ); /* Get the layout of the datatype */ plist = H5Dget_create_plist(*dataset_id); *layout = H5Pget_layout(plist); H5Pclose(plist); return class_id; }
static void getNFilters(const v8::FunctionCallbackInfo<v8::Value>& args) { Filters* obj = ObjectWrap::Unwrap<Filters>(args.This()); hid_t did=H5Dopen(obj->parentId, obj->datasetName.c_str(), H5P_DEFAULT); hid_t pl=H5Dget_create_plist(did); int n=H5Pget_nfilters(pl); H5Pclose(pl); H5Dclose(did); args.GetReturnValue().Set(Int32::New(v8::Isolate::GetCurrent(), n)); }
bool H5Dataset::isChunked() const { hid_t plist = H5Dget_create_plist(dataset); H5D_layout_t layout = H5Pget_layout(plist); bool chunked = layout == H5D_CHUNKED; H5Pclose(plist); return chunked; }
PyObject *get_filter_names( hid_t loc_id, const char *dset_name) { hid_t dset; hid_t dcpl; /* dataset creation property list */ /* hsize_t chsize[64]; /\* chunk size in elements *\/ */ int i, j; int nf; /* number of filters */ unsigned filt_flags; /* filter flags */ size_t cd_nelmts; /* filter client number of values */ unsigned cd_values[20]; /* filter client data values */ char f_name[256]; /* filter name */ PyObject *filters; PyObject *filter_values; /* Open the dataset. */ if ( (dset = H5Dopen( loc_id, dset_name, H5P_DEFAULT )) < 0 ) { goto out; } /* Get the properties container */ dcpl = H5Dget_create_plist(dset); /* Collect information about filters on chunked storage */ if (H5D_CHUNKED==H5Pget_layout(dcpl)) { filters = PyDict_New(); nf = H5Pget_nfilters(dcpl); if ((nf = H5Pget_nfilters(dcpl))>0) { for (i=0; i<nf; i++) { cd_nelmts = 20; H5Pget_filter(dcpl, i, &filt_flags, &cd_nelmts, cd_values, sizeof(f_name), f_name, NULL); filter_values = PyTuple_New(cd_nelmts); for (j=0;j<(long)cd_nelmts;j++) { PyTuple_SetItem(filter_values, j, PyInt_FromLong(cd_values[j])); } PyMapping_SetItemString (filters, f_name, filter_values); } } } else { /* http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52309 */ Py_INCREF(Py_None); filters = Py_None; /* Not chunked, so return None */ } H5Pclose(dcpl); H5Dclose(dset); return filters; out: H5Dclose(dset); Py_INCREF(Py_None); return Py_None; /* Not chunked, so return None */ }
void FWSingle::readInFloat(int step, vector<float>& data_out) { int RANK=1; hid_t dataset; hid_t filespace; hid_t memspace; hid_t cparms; hsize_t dims[2]; /* dataset and chunk dimensions*/ hsize_t chunk_dims[1]; hsize_t col_dims[1]; hsize_t count[2]; hsize_t offset[2]; herr_t status, status_n; int rank, rank_chunk; hsize_t hi, hj; c_file = H5Fopen(fname.str().c_str(),H5F_ACC_RDONLY,H5P_DEFAULT); stringstream gname(""); gname<<"Block_"<< step; hid_t d_file = H5Gopen(c_file,gname.str().c_str()); dataset = H5Dopen(d_file, "Positions"); filespace = H5Dget_space(dataset); rank = H5Sget_simple_extent_ndims(filespace); status_n = H5Sget_simple_extent_dims(filespace, dims, NULL); // if (verbose>1) printf("dataset rank %d, dimensions %lu x %lu\n", rank, (unsigned long)(dims[0]), (unsigned long)(dims[1])); data_out.resize(dims[0]); cparms = H5Dget_create_plist(dataset); if (H5D_CHUNKED == H5Pget_layout(cparms)) { rank_chunk = H5Pget_chunk(cparms, 2, chunk_dims); // if (verbose>1) printf("chunk rank %d, dimensions %lu \n", rank_chunk, (unsigned long)(chunk_dims[0]) ); } memspace = H5Screate_simple(RANK,dims,NULL); status = H5Dread(dataset, H5T_NATIVE_FLOAT, memspace, filespace, H5P_DEFAULT, &(data_out[0])); if(verbose>2) { printf("\n"); printf("Dataset: \n"); for (int j = 0; j < dims[0]; j++) app_log()<<data_out[j]<<" "; app_log()<<endl; } H5Pclose(cparms); H5Dclose(dataset); H5Sclose(filespace); H5Sclose(memspace); H5Gclose(d_file); H5Fclose(c_file); }
HDF5ImageRasterBand::HDF5ImageRasterBand( HDF5ImageDataset *poDSIn, int nBandIn, GDALDataType eType ) : bNoDataSet(false), dfNoDataValue(-9999.0) { poDS = poDSIn; nBand = nBandIn; eDataType = eType; nBlockXSize = poDS->GetRasterXSize( ); nBlockYSize = 1; /* -------------------------------------------------------------------- */ /* Take a copy of Global Metadata since I can't pass Raster */ /* variable to Iterate function. */ /* -------------------------------------------------------------------- */ char **papszMetaGlobal = CSLDuplicate( poDSIn->papszMetadata ); CSLDestroy( poDSIn->papszMetadata ); poDSIn->papszMetadata = NULL; if( poDSIn->poH5Objects->nType == H5G_DATASET ) { poDSIn->CreateMetadata( poDSIn->poH5Objects, H5G_DATASET ); } /* -------------------------------------------------------------------- */ /* Recover Global Metadata and set Band Metadata */ /* -------------------------------------------------------------------- */ SetMetadata( poDSIn->papszMetadata ); CSLDestroy( poDSIn->papszMetadata ); poDSIn->papszMetadata = CSLDuplicate( papszMetaGlobal ); CSLDestroy( papszMetaGlobal ); /* check for chunksize and set it as the blocksize (optimizes read) */ const hid_t listid = H5Dget_create_plist(poDSIn->dataset_id); if (listid>0) { if(H5Pget_layout(listid) == H5D_CHUNKED) { hsize_t panChunkDims[3] = {0, 0, 0}; const int nDimSize = H5Pget_chunk(listid, 3, panChunkDims); CPL_IGNORE_RET_VAL(nDimSize); CPLAssert(nDimSize == poDSIn->ndims); nBlockXSize = (int) panChunkDims[poDSIn->GetXIndex()]; nBlockYSize = (int) panChunkDims[poDSIn->GetYIndex()]; } H5Pclose(listid); } }
HDF5ImageRasterBand::HDF5ImageRasterBand( HDF5ImageDataset *poDS, int nBand, GDALDataType eType ) { char **papszMetaGlobal; this->poDS = poDS; this->nBand = nBand; eDataType = eType; bNoDataSet = FALSE; dfNoDataValue = -9999; nBlockXSize = poDS->GetRasterXSize( ); nBlockYSize = 1; /* -------------------------------------------------------------------- */ /* Take a copy of Global Metadata since I can't pass Raster */ /* variable to Iterate function. */ /* -------------------------------------------------------------------- */ papszMetaGlobal = CSLDuplicate( poDS->papszMetadata ); CSLDestroy( poDS->papszMetadata ); poDS->papszMetadata = NULL; if( poDS->poH5Objects->nType == H5G_DATASET ) { poDS->CreateMetadata( poDS->poH5Objects, H5G_DATASET ); } /* -------------------------------------------------------------------- */ /* Recover Global Metadat and set Band Metadata */ /* -------------------------------------------------------------------- */ SetMetadata( poDS->papszMetadata ); CSLDestroy( poDS->papszMetadata ); poDS->papszMetadata = CSLDuplicate( papszMetaGlobal ); CSLDestroy( papszMetaGlobal ); /* check for chunksize and set it as the blocksize (optimizes read) */ hid_t listid = H5Dget_create_plist(((HDF5ImageDataset * )poDS)->dataset_id); if (listid>0) { if(H5Pget_layout(listid) == H5D_CHUNKED) { hsize_t panChunkDims[3]; int nDimSize = H5Pget_chunk(listid, 3, panChunkDims); nBlockXSize = (int) panChunkDims[nDimSize-1]; nBlockYSize = (int) panChunkDims[nDimSize-2]; } H5Pclose(listid); } }
int_f h5dget_create_plist_c ( hid_t_f *dset_id , hid_t_f *plist_id) /******/ { int ret_value = -1; hid_t c_dset_id; hid_t c_plist_id; c_dset_id = (hid_t)*dset_id; c_plist_id = H5Dget_create_plist(c_dset_id); if(c_plist_id < 0 ) return ret_value; ret_value = 0; *plist_id = (hid_t_f)c_plist_id; return ret_value; }
int H5mdfile::H5_Dopen2(int argc, char **argv, Tcl_Interp *interp) { /* Open an existing dataset */ dataset_id = H5Dopen2(file_id, argv[2], H5P_DEFAULT); // Dataset properties dataspace_id = H5Dget_space(dataset_id); dataset_type_id = H5Dget_type(dataset_id); datatype_size = H5Tget_size(dataset_type_id); dataset_rank = H5Sget_simple_extent_dims(dataspace_id,dims,maxdims); dataset_rank = H5Sget_simple_extent_dims(dataspace_id,dimstotal,maxdims); prop_id = H5Dget_create_plist(dataset_id); if (H5D_CHUNKED == H5Pget_layout(prop_id)) chunk_rank = H5Pget_chunk(prop_id, dataset_rank, chunk_dims); // Dataset size dset_data_size=1; for(int i=0;i<dataset_rank;i++) { dset_data_size*=dims[i]; } return TCL_OK; }
bool BAGRasterBand::Initialize( hid_t hDatasetID, const char *pszName ) { SetDescription( pszName ); this->hDatasetID = hDatasetID; hid_t datatype = H5Dget_type( hDatasetID ); dataspace = H5Dget_space( hDatasetID ); hid_t n_dims = H5Sget_simple_extent_ndims( dataspace ); native = H5Tget_native_type( datatype, H5T_DIR_ASCEND ); hsize_t dims[3], maxdims[3]; eDataType = GH5_GetDataType( native ); if( n_dims == 2 ) { H5Sget_simple_extent_dims( dataspace, dims, maxdims ); nRasterXSize = dims[1]; nRasterYSize = dims[0]; } else { CPLError( CE_Failure, CPLE_AppDefined, "Dataset not of rank 2." ); return false; } nBlockXSize = nRasterXSize; nBlockYSize = 1; /* -------------------------------------------------------------------- */ /* Check for chunksize, and use it as blocksize for optimized */ /* reading. */ /* -------------------------------------------------------------------- */ hid_t listid = H5Dget_create_plist( hDatasetID ); if (listid>0) { if(H5Pget_layout(listid) == H5D_CHUNKED) { hsize_t panChunkDims[3]; int nDimSize = H5Pget_chunk(listid, 3, panChunkDims); nBlockXSize = panChunkDims[nDimSize-1]; nBlockYSize = panChunkDims[nDimSize-2]; } H5Pclose(listid); } /* -------------------------------------------------------------------- */ /* Load min/max information. */ /* -------------------------------------------------------------------- */ if( EQUAL(pszName,"elevation") && GH5_FetchAttribute( hDatasetID, "Maximum Elevation Value", dfMaximum ) && GH5_FetchAttribute( hDatasetID, "Minimum Elevation Value", dfMinimum ) ) bMinMaxSet = true; else if( EQUAL(pszName,"uncertainty") && GH5_FetchAttribute( hDatasetID, "Maximum Uncertainty Value", dfMaximum ) && GH5_FetchAttribute( hDatasetID, "Minimum Uncertainty Value", dfMinimum ) ) bMinMaxSet = true; else if( EQUAL(pszName,"nominal_elevation") && GH5_FetchAttribute( hDatasetID, "max_value", dfMaximum ) && GH5_FetchAttribute( hDatasetID, "min_value", dfMinimum ) ) bMinMaxSet = true; return true; }
bool BAGRasterBand::Initialize( hid_t hDatasetID, const char *pszName ) { SetDescription( pszName ); this->hDatasetID = hDatasetID; hid_t datatype = H5Dget_type( hDatasetID ); dataspace = H5Dget_space( hDatasetID ); int n_dims = H5Sget_simple_extent_ndims( dataspace ); native = H5Tget_native_type( datatype, H5T_DIR_ASCEND ); hsize_t dims[3], maxdims[3]; eDataType = GH5_GetDataType( native ); if( n_dims == 2 ) { H5Sget_simple_extent_dims( dataspace, dims, maxdims ); nRasterXSize = (int) dims[1]; nRasterYSize = (int) dims[0]; } else { CPLError( CE_Failure, CPLE_AppDefined, "Dataset not of rank 2." ); return false; } nBlockXSize = nRasterXSize; nBlockYSize = 1; /* -------------------------------------------------------------------- */ /* Check for chunksize, and use it as blocksize for optimized */ /* reading. */ /* -------------------------------------------------------------------- */ hid_t listid = H5Dget_create_plist( hDatasetID ); if (listid>0) { if(H5Pget_layout(listid) == H5D_CHUNKED) { hsize_t panChunkDims[3]; int nDimSize = H5Pget_chunk(listid, 3, panChunkDims); nBlockXSize = (int) panChunkDims[nDimSize-1]; nBlockYSize = (int) panChunkDims[nDimSize-2]; } int nfilters = H5Pget_nfilters( listid ); H5Z_filter_t filter; char name[120]; size_t cd_nelmts = 20; unsigned int cd_values[20]; unsigned int flags; for (int i = 0; i < nfilters; i++) { filter = H5Pget_filter(listid, i, &flags, (size_t *)&cd_nelmts, cd_values, 120, name); if (filter == H5Z_FILTER_DEFLATE) poDS->SetMetadataItem( "COMPRESSION", "DEFLATE", "IMAGE_STRUCTURE" ); else if (filter == H5Z_FILTER_NBIT) poDS->SetMetadataItem( "COMPRESSION", "NBIT", "IMAGE_STRUCTURE" ); else if (filter == H5Z_FILTER_SCALEOFFSET) poDS->SetMetadataItem( "COMPRESSION", "SCALEOFFSET", "IMAGE_STRUCTURE" ); else if (filter == H5Z_FILTER_SZIP) poDS->SetMetadataItem( "COMPRESSION", "SZIP", "IMAGE_STRUCTURE" ); } H5Pclose(listid); } /* -------------------------------------------------------------------- */ /* Load min/max information. */ /* -------------------------------------------------------------------- */ if( EQUAL(pszName,"elevation") && GH5_FetchAttribute( hDatasetID, "Maximum Elevation Value", dfMaximum ) && GH5_FetchAttribute( hDatasetID, "Minimum Elevation Value", dfMinimum ) ) bMinMaxSet = true; else if( EQUAL(pszName,"uncertainty") && GH5_FetchAttribute( hDatasetID, "Maximum Uncertainty Value", dfMaximum ) && GH5_FetchAttribute( hDatasetID, "Minimum Uncertainty Value", dfMinimum ) ) { /* Some products where uncertainty band is completely set to nodata */ /* wrongly declare minimum and maximum to 0.0 */ if( dfMinimum != 0.0 && dfMaximum != 0.0 ) bMinMaxSet = true; } else if( EQUAL(pszName,"nominal_elevation") && GH5_FetchAttribute( hDatasetID, "max_value", dfMaximum ) && GH5_FetchAttribute( hDatasetID, "min_value", dfMinimum ) ) bMinMaxSet = true; return true; }
/*! Get a copy of the volume property list. When the program is finished * using the property list it should call mifree_volume_props() to free the * memory associated with the list. * \param volume A volume handle * \param props A pointer to the returned volume properties handle. * \ingroup mi2VPrp */ int miget_volume_props(mihandle_t volume, mivolumeprops_t *props) { mivolumeprops_t handle; hid_t hdf_vol_dataset; hid_t hdf_plist; int nfilters; unsigned int flags; size_t cd_nelmts; unsigned int cd_values[MI2_MAX_CD_ELEMENTS]; char fname[MI2_CHAR_LENGTH]; int fcode; if (volume->hdf_id < 0) { return (MI_ERROR); } hdf_vol_dataset = midescend_path(volume->hdf_id, "/minc-2.0/image/0/image"); if (hdf_vol_dataset < 0) { return (MI_ERROR); } hdf_plist = H5Dget_create_plist(hdf_vol_dataset); if (hdf_plist < 0) { return (MI_ERROR); } handle = (mivolumeprops_t)malloc(sizeof(struct mivolprops)); if (handle == NULL) { return (MI_ERROR); } /* Get the layout of the raw data for a dataset. */ if (H5Pget_layout(hdf_plist) == H5D_CHUNKED) { hsize_t dims[MI2_MAX_VAR_DIMS]; int i; /* Returns chunk dimensionality */ handle->edge_count = H5Pget_chunk(hdf_plist, MI2_MAX_VAR_DIMS, dims); if (handle->edge_count < 0) { free(handle); return (MI_ERROR); } handle->edge_lengths = (int *)malloc(handle->edge_count*sizeof(int)); if (handle->edge_lengths == NULL) { free(handle); return (MI_ERROR); } for (i = 0; i < handle->edge_count; i++) { handle->edge_lengths[i] = dims[i]; } /* Get the number of filters in the pipeline */ nfilters = H5Pget_nfilters(hdf_plist); if (nfilters == 0) { handle->zlib_level = 0; handle->compression_type = MI_COMPRESS_NONE; } else { for (i = 0; i < nfilters; i++) { cd_nelmts = MI2_MAX_CD_ELEMENTS; fcode = H5Pget_filter1(hdf_plist, i, &flags, &cd_nelmts, cd_values, sizeof(fname), fname); switch (fcode) { case H5Z_FILTER_DEFLATE: handle->compression_type = MI_COMPRESS_ZLIB; handle->zlib_level = cd_values[0]; break; case H5Z_FILTER_SHUFFLE: break; case H5Z_FILTER_FLETCHER32: break; case H5Z_FILTER_SZIP: break; default: break; } } } } else { handle->edge_count = 0; handle->edge_lengths = NULL; handle->zlib_level = 0; handle->compression_type = MI_COMPRESS_NONE; } *props = handle; H5Pclose(hdf_plist); H5Dclose(hdf_vol_dataset); return (MI_NOERROR); }
int h5repack_cmpdcpl(const char *fname1, const char *fname2) { hid_t fid1=-1; /* file ID */ hid_t fid2=-1; /* file ID */ hid_t dset1=-1; /* dataset ID */ hid_t dset2=-1; /* dataset ID */ hid_t dcpl1=-1; /* dataset creation property list ID */ hid_t dcpl2=-1; /* dataset creation property list ID */ trav_table_t *travt1=NULL; trav_table_t *travt2=NULL; int ret=1; unsigned int i; /*------------------------------------------------------------------------- * open the files first; if they are not valid, no point in continuing *------------------------------------------------------------------------- */ /* disable error reporting */ H5E_BEGIN_TRY { /* Open the files */ if ((fid1=H5Fopen(fname1,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ) { error_msg(progname, "<%s>: %s\n", fname1, H5FOPENERROR ); return -1; } if ((fid2=H5Fopen(fname2,H5F_ACC_RDONLY,H5P_DEFAULT))<0 ) { error_msg(progname, "<%s>: %s\n", fname2, H5FOPENERROR ); H5Fclose(fid1); return -1; } /* enable error reporting */ } H5E_END_TRY; /*------------------------------------------------------------------------- * get file table list of objects *------------------------------------------------------------------------- */ trav_table_init(&travt1); trav_table_init(&travt2); if (h5trav_gettable(fid1,travt1)<0) goto error; if (h5trav_gettable(fid2,travt2)<0) goto error; /*------------------------------------------------------------------------- * traverse the suppplied object list *------------------------------------------------------------------------- */ for ( i=0; i < travt1->nobjs; i++) { switch ( travt1->objs[i].type ) { /*------------------------------------------------------------------------- * nothing to do for groups, links and types *------------------------------------------------------------------------- */ default: break; /*------------------------------------------------------------------------- * H5G_DATASET *------------------------------------------------------------------------- */ case H5G_DATASET: if ((dset1=H5Dopen(fid1,travt1->objs[i].name))<0) goto error; if ((dset2=H5Dopen(fid2,travt1->objs[i].name))<0) goto error; if ((dcpl1=H5Dget_create_plist(dset1))<0) goto error; if ((dcpl2=H5Dget_create_plist(dset2))<0) goto error; /*------------------------------------------------------------------------- * compare the property lists *------------------------------------------------------------------------- */ if ((ret=H5Pequal(dcpl1,dcpl2))<0) goto error; if (ret==0) { error_msg(progname, "property lists for <%s> are different\n",travt1->objs[i].name); goto error; } /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ if (H5Pclose(dcpl1)<0) goto error; if (H5Pclose(dcpl2)<0) goto error; if (H5Dclose(dset1)<0) goto error; if (H5Dclose(dset2)<0) goto error; break; } /*switch*/ } /*i*/ /*------------------------------------------------------------------------- * free *------------------------------------------------------------------------- */ trav_table_free(travt1); trav_table_free(travt2); /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ H5Fclose(fid1); H5Fclose(fid2); return ret; /*------------------------------------------------------------------------- * error *------------------------------------------------------------------------- */ error: H5E_BEGIN_TRY { H5Pclose(dcpl1); H5Pclose(dcpl2); H5Dclose(dset1); H5Dclose(dset2); H5Fclose(fid1); H5Fclose(fid2); trav_table_free(travt1); trav_table_free(travt2); } H5E_END_TRY; return -1; }
int main (void) { hid_t vfile, file, src_space, mem_space, vspace, vdset, dset; /* Handles */ hid_t dcpl, dapl; herr_t status; hsize_t vdsdims[3] = {4*DIM0_1, VDSDIM1, VDSDIM2}, vdsdims_max[3] = {VDSDIM0, VDSDIM1, VDSDIM2}, dims[3] = {DIM0_1, DIM1, DIM2}, memdims[3] = {DIM0_1, DIM1, DIM2}, extdims[3] = {0, DIM1, DIM2}, /* Dimensions of the extended source datasets */ chunk_dims[3] = {DIM0_1, DIM1, DIM2}, dims_max[3] = {DIM0, DIM1, DIM2}, vdsdims_out[3], vdsdims_max_out[3], start[3], /* Hyperslab parameters */ stride[3], count[3], src_count[3], block[3]; hsize_t start_out[3], /* Hyperslab parameter out */ stride_out[3], count_out[3], block_out[3]; int i, j; H5D_layout_t layout; /* Storage layout */ size_t num_map; /* Number of mappings */ ssize_t len; /* Length of the string; also a return value */ char *filename; char *dsetname; int wdata[DIM0_1*DIM1*DIM2]; /* * Create source files and datasets. This step is optional. */ for (i=0; i < PLANE_STRIDE; i++) { /* * Initialize data for i-th source dataset. */ for (j = 0; j < DIM0_1*DIM1*DIM2; j++) wdata[j] = i+1; /* * Create the source files and datasets. Write data to each dataset and * close all resources. */ file = H5Fcreate (SRC_FILE[i], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); src_space = H5Screate_simple (RANK, dims, dims_max); dcpl = H5Pcreate(H5P_DATASET_CREATE); status = H5Pset_chunk (dcpl, RANK, chunk_dims); dset = H5Dcreate2 (file, SRC_DATASET[i], H5T_NATIVE_INT, src_space, H5P_DEFAULT, dcpl, H5P_DEFAULT); status = H5Dwrite (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata); status = H5Sclose (src_space); status = H5Pclose (dcpl); status = H5Dclose (dset); status = H5Fclose (file); } vfile = H5Fcreate (VFILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); /* Create VDS dataspace. */ vspace = H5Screate_simple (RANK, vdsdims, vdsdims_max); /* Create dataspaces for the source dataset. */ src_space = H5Screate_simple (RANK, dims, dims_max); /* Create VDS creation property */ dcpl = H5Pcreate (H5P_DATASET_CREATE); /* Initialize hyperslab values */ start[0] = 0; start[1] = 0; start[2] = 0; stride[0] = PLANE_STRIDE; /* we will select every fifth plane in VDS */ stride[1] = 1; stride[2] = 1; count[0] = H5S_UNLIMITED; count[1] = 1; count[2] = 1; src_count[0] = H5S_UNLIMITED; src_count[1] = 1; src_count[2] = 1; block[0] = 1; block[1] = DIM1; block[2] = DIM2; /* * Build the mappings * */ status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, src_count, block); for (i=0; i < PLANE_STRIDE; i++) { status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, stride, count, block); status = H5Pset_virtual (dcpl, vspace, SRC_FILE[i], SRC_DATASET[i], src_space); start[0]++; } H5Sselect_none(vspace); /* Create a virtual dataset */ vdset = H5Dcreate2 (vfile, DATASET, H5T_NATIVE_INT, vspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); status = H5Sclose (vspace); status = H5Sclose (src_space); status = H5Pclose (dcpl); /* Let's add data to the source datasets and check new dimensions for VDS */ /* We will add only one plane to the first source dataset, two planes to the second one, three to the third, and four to the forth. */ for (i=0; i < PLANE_STRIDE; i++) { /* * Initialize data for i-th source dataset. */ for (j = 0; j < (i+1)*DIM1*DIM2; j++) wdata[j] = 10*(i+1); /* * Open the source files and datasets. Appen data to each dataset and * close all resources. */ file = H5Fopen (SRC_FILE[i], H5F_ACC_RDWR, H5P_DEFAULT); dset = H5Dopen2 (file, SRC_DATASET[i], H5P_DEFAULT); extdims[0] = DIM0_1+i+1; status = H5Dset_extent (dset, extdims); src_space = H5Dget_space (dset); start[0] = DIM0_1; start[1] = 0; start[2] = 0; count[0] = 1; count[1] = 1; count[2] = 1; block[0] = i+1; block[1] = DIM1; block[2] = DIM2; memdims[0] = i+1; mem_space = H5Screate_simple(RANK, memdims, NULL); status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, count, block); status = H5Dwrite (dset, H5T_NATIVE_INT, mem_space, src_space, H5P_DEFAULT, wdata); status = H5Sclose (src_space); status = H5Dclose (dset); status = H5Fclose (file); } status = H5Dclose (vdset); status = H5Fclose (vfile); /* * Now we begin the read section of this example. */ /* * Open file and dataset using the default properties. */ vfile = H5Fopen (VFILE, H5F_ACC_RDONLY, H5P_DEFAULT); /* * Open VDS using different access properties to use max or * min extents depending on the sizes of the underlying datasets */ dapl = H5Pcreate (H5P_DATASET_ACCESS); for(i = 0; i < 2; i++) { status = H5Pset_virtual_view (dapl, i ? H5D_VDS_LAST_AVAILABLE : H5D_VDS_FIRST_MISSING); vdset = H5Dopen2 (vfile, DATASET, dapl); /* Let's get space of the VDS and its dimension; we should get 32(or 20)x10x10 */ vspace = H5Dget_space (vdset); H5Sget_simple_extent_dims (vspace, vdsdims_out, vdsdims_max_out); printf ("VDS dimensions, bounds = H5D_VDS_%s: ", i ? "LAST_AVAILABLE" : "FIRST_MISSING"); for (j=0; j<RANK; j++) printf (" %d ", (int)vdsdims_out[j]); printf ("\n"); /* Close */ status = H5Dclose (vdset); status = H5Sclose (vspace); } status = H5Pclose (dapl); vdset = H5Dopen2 (vfile, DATASET, H5P_DEFAULT); /* * Get creation property list and mapping properties. */ dcpl = H5Dget_create_plist (vdset); /* * Get storage layout. */ layout = H5Pget_layout (dcpl); if (H5D_VIRTUAL == layout) printf(" Dataset has a virtual layout \n"); else printf(" Wrong layout found \n"); /* * Find number of mappings. */ status = H5Pget_virtual_count (dcpl, &num_map); printf(" Number of mappings is %lu\n", (unsigned long)num_map); /* * Get mapping parameters for each mapping. */ for (i = 0; i < (int)num_map; i++) { printf(" Mapping %d \n", i); printf(" Selection in the virtual dataset \n"); /* Get selection in the virttual dataset */ vspace = H5Pget_virtual_vspace (dcpl, (size_t)i); if (H5Sget_select_type(vspace) == H5S_SEL_HYPERSLABS) { if (H5Sis_regular_hyperslab(vspace)) { status = H5Sget_regular_hyperslab (vspace, start_out, stride_out, count_out, block_out); printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]); printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]); printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]); printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]); } } /* Get source file name */ len = H5Pget_virtual_filename (dcpl, (size_t)i, NULL, 0); filename = (char *)malloc((size_t)len*sizeof(char)+1); H5Pget_virtual_filename (dcpl, (size_t)i, filename, len+1); printf(" Source filename %s\n", filename); /* Get source dataset name */ len = H5Pget_virtual_dsetname (dcpl, (size_t)i, NULL, 0); dsetname = (char *)malloc((size_t)len*sizeof(char)+1); H5Pget_virtual_dsetname (dcpl, (size_t)i, dsetname, len+1); printf(" Source dataset name %s\n", dsetname); /* Get selection in the source dataset */ printf(" Selection in the source dataset \n"); src_space = H5Pget_virtual_srcspace (dcpl, (size_t)i); if (H5Sget_select_type(src_space) == H5S_SEL_HYPERSLABS) { if (H5Sis_regular_hyperslab(src_space)) { status = H5Sget_regular_hyperslab (src_space, start_out, stride_out, count_out, block_out); printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]); printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]); printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]); printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]); } } H5Sclose(vspace); H5Sclose(src_space); free(filename); free(dsetname); } /* * Close and release resources. */ status = H5Pclose (dcpl); status = H5Dclose (vdset); status = H5Fclose (vfile); return 0; }
int main (void) { hid_t file, space, src_space, vspace, dset; /* Handles */ hid_t dcpl; herr_t status; hsize_t vdsdims[2] = {DIM0, DIM1}, /* Virtual dataset dimension */ dims[2] = {DIM0, DIM1}; /* Source dataset dimensions */ int wdata[DIM0][DIM1], /* Write buffer for source dataset */ rdata[DIM0][DIM1], /* Read buffer for virtual dataset */ i, j; H5D_layout_t layout; /* Storage layout */ size_t num_map; /* Number of mappings */ ssize_t len; /* Length of the string; also a return value */ char *filename; char *dsetname; /* * Initialize data. */ for (i = 0; i < DIM0; i++) for (j = 0; j < DIM1; j++) wdata[i][j] = i+1; /* * Create the source file and the dataset. Write data to the source dataset * and close all resources. */ file = H5Fcreate (SRC_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); space = H5Screate_simple (RANK, dims, NULL); dset = H5Dcreate2 (file, SRC_DATASET, H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); status = H5Dwrite (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]); status = H5Sclose (space); status = H5Dclose (dset); status = H5Fclose (file); /* Create file in which virtual dataset will be stored. */ file = H5Fcreate (FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); /* Create VDS dataspace. */ vspace = H5Screate_simple (RANK, vdsdims, NULL); /* Set VDS creation property. */ dcpl = H5Pcreate (H5P_DATASET_CREATE); /* * Build the mappings. * Selections in the source datasets are H5S_ALL. * In the virtual dataset we select the first, the second and the third rows * and map each row to the data in the corresponding source dataset. */ src_space = H5Screate_simple (RANK, dims, NULL); status = H5Pset_virtual (dcpl, vspace, SRC_FILE, SRC_DATASET, src_space); /* Create a virtual dataset. */ dset = H5Dcreate2 (file, DATASET, H5T_NATIVE_INT, vspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); status = H5Sclose (vspace); status = H5Sclose (src_space); status = H5Dclose (dset); status = H5Fclose (file); /* * Now we begin the read section of this example. */ /* * Open the file and virtual dataset. */ file = H5Fopen (FILE, H5F_ACC_RDONLY, H5P_DEFAULT); dset = H5Dopen2 (file, DATASET, H5P_DEFAULT); /* * Get creation property list and mapping properties. */ dcpl = H5Dget_create_plist (dset); /* * Get storage layout. */ layout = H5Pget_layout (dcpl); if (H5D_VIRTUAL == layout) printf(" Dataset has a virtual layout \n"); else printf(" Wrong layout found \n"); /* * Find the number of mappings. */ status = H5Pget_virtual_count (dcpl, &num_map); printf(" Number of mappings is %lu\n", (unsigned long)num_map); /* * Get mapping parameters for each mapping. */ for (i = 0; i < (int)num_map; i++) { printf(" Mapping %d \n", i); printf(" Selection in the virtual dataset "); /* Get selection in the virttual dataset */ vspace = H5Pget_virtual_vspace (dcpl, (size_t)i); /* Make sure it is ALL selection and then print selection. */ if(H5Sget_select_type(vspace) == H5S_SEL_ALL) { printf("Selection is H5S_ALL \n"); } /* Get source file name. */ len = H5Pget_virtual_filename (dcpl, (size_t)i, NULL, 0); filename = (char *)malloc((size_t)len*sizeof(char)+1); H5Pget_virtual_filename (dcpl, (size_t)i, filename, len+1); printf(" Source filename %s\n", filename); /* Get source dataset name. */ len = H5Pget_virtual_dsetname (dcpl, (size_t)i, NULL, 0); dsetname = (char *)malloc((size_t)len*sizeof(char)+1); H5Pget_virtual_dsetname (dcpl, (size_t)i, dsetname, len+1); printf(" Source dataset name %s\n", dsetname); /* Get selection in the source dataset. */ printf(" Selection in the source dataset "); src_space = H5Pget_virtual_srcspace (dcpl, (size_t)i); /* Make sure it is ALL selection and then print selection. */ if(H5Sget_select_type(src_space) == H5S_SEL_ALL) { printf("Selection is H5S_ALL \n"); } H5Sclose(vspace); H5Sclose(src_space); free(filename); free(dsetname); } /* * Read the data using the default properties. */ status = H5Dread (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata[0]); /* * Output the data to the screen. */ printf (" VDS Data:\n"); for (i=0; i<DIM0; i++) { printf (" ["); for (j=0; j<DIM1; j++) printf (" %3d", rdata[i][j]); printf ("]\n"); } /* * Close and release resources. */ status = H5Pclose (dcpl); status = H5Dclose (dset); status = H5Fclose (file); return 0; }
DataSetCreatePropList DataSet::createPropList () const { return DataSetCreatePropList (Exception::check ("H5Dget_create_plist", H5Dget_create_plist (handle ()))); }
/*------------------------------------------------------------------------- * Function: check_dataset * * Purpose: To check whether a dataset can be monitored: A chunked dataset with unlimited or max. dimension setting * * Return: Non-negative on success: dataset can be monitored * Negative on failure: dataset cannot be monitored * * Programmer: Vailin Choi; August 2010 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t check_dataset(hid_t fid, char *dsetname) { hid_t did=-1; /* Dataset id */ hid_t dcp=-1; /* Dataset creation property */ hid_t sid=-1; /* Dataset's dataspace id */ int ndims; /* # of dimensions in the dataspace */ unsigned u; /* Local index variable */ hsize_t cur_dims[H5S_MAX_RANK]; /* size of dataspace dimensions */ hsize_t max_dims[H5S_MAX_RANK]; /* maximum size of dataspace dimensions */ hbool_t unlim_max_dims = FALSE; /* whether dataset has unlimited or max. dimension setting */ void *edata; H5E_auto2_t func; herr_t ret_value = SUCCEED; /* Return value */ /* Disable error reporting */ H5Eget_auto2(H5E_DEFAULT, &func, &edata); H5Eset_auto2(H5E_DEFAULT, NULL, NULL); /* Open the dataset */ if((did = H5Dopen2(fid, dsetname, H5P_DEFAULT)) < 0) { error_msg("unable to open dataset \"%s\"\n", dsetname); ret_value = FAIL; goto done; } /* Get dataset's creation property list */ if((dcp = H5Dget_create_plist(did)) < 0) { error_msg("unable to get dataset's creation property list\"%s\"\n", dsetname); ret_value = FAIL; goto done; } /* Query dataset's layout; the layout should be chunked */ if(H5Pget_layout(dcp) != H5D_CHUNKED) { error_msg("\"%s\" should be a chunked dataset\n", dsetname); ret_value = FAIL; goto done; } HDmemset(cur_dims, 0, sizeof cur_dims); HDmemset(max_dims, 0, sizeof max_dims); /* Get dataset's dataspace */ if((sid = H5Dget_space(did)) < 0) { error_msg("can't get dataset's dataspace\"%s\"\n", dsetname); ret_value = FAIL; goto done; } /* Get dimension size of dataset's dataspace */ if((ndims = H5Sget_simple_extent_dims(sid, cur_dims, max_dims)) < 0) { error_msg("can't get dataspace dimensions for dataset \"%s\"\n", dsetname); ret_value = FAIL; goto done; } /* Check whether dataset has unlimited dimension or max. dimension setting */ for(u = 0; u < (unsigned)ndims; u++) if(max_dims[u] == H5S_UNLIMITED || cur_dims[u] != max_dims[u]) { unlim_max_dims = TRUE; break; } if(!unlim_max_dims) { error_msg("\"%s\" should have unlimited or max. dimension setting\n", dsetname); ret_value = FAIL; } done: H5Eset_auto2(H5E_DEFAULT, func, edata); /* Closing */ H5E_BEGIN_TRY H5Sclose(sid); H5Pclose(dcp); H5Dclose(did); H5E_END_TRY return(ret_value); } /* check_dataset() */
/*------------------------------------------------------------------------- * Function: dataset_stats * * Purpose: Gather statistics about the dataset * * Return: Success: 0 * * Failure: -1 * * Programmer: Quincey Koziol * Tuesday, August 16, 2005 * *------------------------------------------------------------------------- */ static herr_t dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) { unsigned bin; /* "bin" the number of objects falls in */ hid_t did; /* Dataset ID */ hid_t sid; /* Dataspace ID */ hid_t tid; /* Datatype ID */ hid_t dcpl; /* Dataset creation property list ID */ hsize_t dims[H5S_MAX_RANK];/* Dimensions of dataset */ H5D_layout_t lout; /* Layout of dataset */ unsigned type_found; /* Whether the dataset's datatype was */ /* already found */ int ndims; /* Number of dimensions of dataset */ hsize_t storage; /* Size of dataset storage */ unsigned u; /* Local index variable */ int num_ext; /* Number of external files for a dataset */ int nfltr; /* Number of filters for a dataset */ H5Z_filter_t fltr; /* Filter identifier */ herr_t ret; /* Gather statistics about this type of object */ iter->uniq_dsets++; /* Get object header information */ iter->dset_ohdr_info.total_size += oi->hdr.space.total; iter->dset_ohdr_info.free_size += oi->hdr.space.free; did = H5Dopen2(iter->fid, name, H5P_DEFAULT); HDassert(did > 0); /* Update dataset metadata info */ iter->datasets_index_storage_size += oi->meta_size.obj.index_size; iter->datasets_heap_storage_size += oi->meta_size.obj.heap_size; /* Update attribute metadata info */ ret = attribute_stats(iter, oi); HDassert(ret >= 0); /* Get storage info */ storage = H5Dget_storage_size(did); /* Gather layout statistics */ dcpl = H5Dget_create_plist(did); HDassert(dcpl > 0); lout = H5Pget_layout(dcpl); HDassert(lout >= 0); /* Object header's total size for H5D_COMPACT layout includes raw data size */ /* "storage" also includes H5D_COMPACT raw data size */ if(lout == H5D_COMPACT) iter->dset_ohdr_info.total_size -= storage; /* Track the layout type for dataset */ (iter->dset_layouts[lout])++; /* Get the number of external files for the dataset */ num_ext = H5Pget_external_count(dcpl); assert (num_ext >= 0); /* Accumulate raw data size accordingly */ if(num_ext) { iter->nexternal += (unsigned long)num_ext; iter->dset_external_storage_size += (unsigned long)storage; } else iter->dset_storage_size += storage; /* Gather dataspace statistics */ sid = H5Dget_space(did); HDassert(sid > 0); ndims = H5Sget_simple_extent_dims(sid, dims, NULL); HDassert(ndims >= 0); /* Check for larger rank of dataset */ if((unsigned)ndims > iter->max_dset_rank) iter->max_dset_rank = (unsigned)ndims; /* Track the number of datasets with each rank */ (iter->dset_rank_count[ndims])++; /* Only gather dim size statistics on 1-D datasets */ if(ndims == 1) { iter->max_dset_dims = dims[0]; if(dims[0] < SIZE_SMALL_DSETS) (iter->small_dset_dims[(size_t)dims[0]])++; /* Add dim count to proper bin */ bin = ceil_log10((unsigned long)dims[0]); if((bin + 1) > iter->dset_dim_nbins) { /* Allocate more storage for info about dataset's datatype */ iter->dset_dim_bins = (unsigned long *)HDrealloc(iter->dset_dim_bins, (bin + 1) * sizeof(unsigned long)); HDassert(iter->dset_dim_bins); /* Initialize counts for intermediate bins */ while(iter->dset_dim_nbins < bin) iter->dset_dim_bins[iter->dset_dim_nbins++] = 0; iter->dset_dim_nbins++; /* Initialize count for this bin */ iter->dset_dim_bins[bin] = 1; } /* end if */ else (iter->dset_dim_bins[bin])++; } /* end if */ ret = H5Sclose(sid); HDassert(ret >= 0); /* Gather datatype statistics */ tid = H5Dget_type(did); HDassert(tid > 0); type_found = FALSE; for(u = 0; u < iter->dset_ntypes; u++) if(H5Tequal(iter->dset_type_info[u].tid, tid) > 0) { type_found = TRUE; break; } /* end for */ if(type_found) (iter->dset_type_info[u].count)++; else { unsigned curr_ntype = iter->dset_ntypes; /* Increment # of datatypes seen for datasets */ iter->dset_ntypes++; /* Allocate more storage for info about dataset's datatype */ iter->dset_type_info = (dtype_info_t *)HDrealloc(iter->dset_type_info, iter->dset_ntypes * sizeof(dtype_info_t)); HDassert(iter->dset_type_info); /* Initialize information about datatype */ iter->dset_type_info[curr_ntype].tid = H5Tcopy(tid); HDassert(iter->dset_type_info[curr_ntype].tid > 0); iter->dset_type_info[curr_ntype].count = 1; iter->dset_type_info[curr_ntype].named = 0; /* Set index for later */ u = curr_ntype; } /* end else */ /* Check if the datatype is a named datatype */ if(H5Tcommitted(tid) > 0) (iter->dset_type_info[u].named)++; ret = H5Tclose(tid); HDassert(ret >= 0); /* Track different filters */ if((nfltr = H5Pget_nfilters(dcpl)) >= 0) { if(nfltr == 0) iter->dset_comptype[0]++; for(u = 0; u < (unsigned)nfltr; u++) { fltr = H5Pget_filter2(dcpl, u, 0, 0, 0, 0, 0, NULL); if(fltr >= 0) { if(fltr < (H5_NFILTERS_IMPL - 1)) iter->dset_comptype[fltr]++; else iter->dset_comptype[H5_NFILTERS_IMPL - 1]++; /*other filters*/ } /* end if */ } /* end for */ } /* endif nfltr */ ret = H5Pclose(dcpl); HDassert(ret >= 0); ret = H5Dclose(did); HDassert(ret >= 0); return 0; } /* end dataset_stats() */
int h5repack_verify(const char *out_fname, pack_opt_t *options) { int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ hid_t fidout = -1; /* file ID for output file*/ hid_t did = -1; /* dataset ID */ hid_t pid = -1; /* dataset creation property list ID */ hid_t sid = -1; /* space ID */ hid_t tid = -1; /* type ID */ unsigned int i; trav_table_t *travt = NULL; int ok = 1; /* open the output file */ if((fidout = H5Fopen(out_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) return -1; for(i = 0; i < options->op_tbl->nelems; i++) { char* name = options->op_tbl->objs[i].path; pack_info_t *obj = &options->op_tbl->objs[i]; /*------------------------------------------------------------------------- * open *------------------------------------------------------------------------- */ if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); if((sid = H5Dget_space(did)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed"); if((pid = H5Dget_create_plist(did)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); if((tid = H5Dget_type(did)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); /*------------------------------------------------------------------------- * filter check *------------------------------------------------------------------------- */ if(verify_filters(pid, tid, obj->nfilters, obj->filter) <= 0) ok = 0; /*------------------------------------------------------------------------- * layout check *------------------------------------------------------------------------- */ if((obj->layout != -1) && (verify_layout(pid, obj) == 0)) ok = 0; /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ if(H5Pclose(pid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); if (H5Sclose(sid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); if (H5Dclose(did) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); if (H5Tclose(tid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); } /*------------------------------------------------------------------------- * check for the "all" objects option *------------------------------------------------------------------------- */ if(options->all_filter == 1 || options->all_layout == 1) { /* init table */ trav_table_init(&travt); /* get the list of objects in the file */ if(h5trav_gettable(fidout, travt) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed"); for(i = 0; i < travt->nobjs; i++) { char *name = travt->objs[i].name; if(travt->objs[i].type == H5TRAV_TYPE_DATASET) { /*------------------------------------------------------------------------- * open *------------------------------------------------------------------------- */ if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); if((sid = H5Dget_space(did)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed"); if((pid = H5Dget_create_plist(did)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); if((tid = H5Dget_type(did)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed"); /*------------------------------------------------------------------------- * filter check *------------------------------------------------------------------------- */ if(options->all_filter == 1) { if(verify_filters(pid, tid, options->n_filter_g, options->filter_g) <= 0) ok = 0; } /*------------------------------------------------------------------------- * layout check *------------------------------------------------------------------------- */ if(options->all_layout == 1) { pack_info_t pack; init_packobject(&pack); pack.layout = options->layout_g; pack.chunk = options->chunk_g; if(verify_layout(pid, &pack) == 0) ok = 0; } /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ if (H5Pclose(pid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); if (H5Sclose(sid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); if (H5Dclose(did) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); if (H5Tclose(tid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); } /* if */ } /* i */ /* free table */ trav_table_free(travt); } /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ if (H5Fclose(fidout) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fclose failed"); return ok; done: H5E_BEGIN_TRY { H5Pclose(pid); H5Sclose(sid); H5Dclose(did); H5Fclose(fidout); if (travt) trav_table_free(travt); } H5E_END_TRY; return ret_value; } /* h5repack_verify() */
/*------------------------------------------------------------------------- * test_compress * * Ensures that a FL packet table can be compressed. * This test creates a file named TEST_COMPRESS_FILE * *------------------------------------------------------------------------- */ static int test_compress(void) { hid_t fid1 = -1; herr_t err; hid_t table = -1; hid_t part_t = -1; hid_t dset_id = -1; hid_t plist_id = -1; size_t c; size_t num_elems = 1; unsigned filter_vals[1]; particle_t readPart[1]; hsize_t count; TESTING("packet table compression"); /* Create a file. */ if((fid1 = H5Fcreate(TEST_COMPRESS_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; /* Create a datatype for the particle struct */ part_t = make_particle_type(); HDassert(part_t != -1); /* Create a new table with compression level 8 */ table = H5PTcreate_fl(fid1, "Compressed Test Dataset", part_t, (hsize_t)80, 8); if( H5PTis_valid(table) < 0) TEST_ERROR; /* We can now use this table exactly the same way we use a normal uncompressed * packet table, and it should pass the same tests. */ /* Add many particles */ for(c = 0; c < BIG_TABLE_SIZE ; c+=8) { /* Append eight particles at once*/ err = H5PTappend(table, (size_t)8, &(testPart[0])); if( err < 0) TEST_ERROR; } /* Count the number of packets in the table */ err = H5PTget_num_packets(table, &count); if( err < 0) TEST_ERROR; if( count != BIG_TABLE_SIZE ) TEST_ERROR; /* Read particles to ensure that all of them were written correctly */ HDmemset(readPart, 0, sizeof(readPart)); for(c = 0; c < BIG_TABLE_SIZE; c++) { err = H5PTget_next(table, 1, readPart); if(err < 0) TEST_ERROR; /* Ensure that particles were read correctly */ if( cmp_par(c % 8, 0, testPart, readPart) != 0) TEST_ERROR; } /* Close the table */ err = H5PTclose(table); if( err < 0) TEST_ERROR; /* Open the packet table as a regular dataset and make sure that the * compression filter is set. */ dset_id = H5Dopen2(fid1, "Compressed Test Dataset", H5P_DEFAULT); if( dset_id < 0) TEST_ERROR; plist_id = H5Dget_create_plist(dset_id); if( plist_id < 0) TEST_ERROR; err = H5Pget_filter_by_id2(plist_id, H5Z_FILTER_DEFLATE, NULL, &num_elems, filter_vals, 0, NULL, NULL); if( err < 0) TEST_ERROR; /* The compression level should be 8, the value we passed in */ if(filter_vals[0] != 8) TEST_ERROR; /* Clean up */ err = H5Pclose(plist_id); if( err < 0) TEST_ERROR; err = H5Dclose(dset_id); if( err < 0) TEST_ERROR; /* Create a new table without compression. */ table = H5PTcreate_fl(fid1, "Uncompressed Dataset", part_t, (hsize_t)80, -1); if(table < 0) TEST_ERROR; /* Close the packet table */ err = H5PTclose(table); if( err < 0) TEST_ERROR; /* Open the packet table as a regular dataset and make sure that the * compression filter is not set. */ dset_id = H5Dopen2(fid1, "Uncompressed Dataset", H5P_DEFAULT); if( dset_id < 0) TEST_ERROR; plist_id = H5Dget_create_plist(dset_id); if( plist_id < 0) TEST_ERROR; H5E_BEGIN_TRY { err = H5Pget_filter_by_id2(plist_id, H5Z_FILTER_DEFLATE, NULL, &num_elems, filter_vals, 0, NULL, NULL); if( err >= 0) TEST_ERROR; } H5E_END_TRY /* Clean up */ err = H5Pclose(plist_id); if( err < 0) TEST_ERROR; err = H5Dclose(dset_id); if( err < 0) TEST_ERROR; /* Close the datatype and the file */ err = H5Tclose(part_t); if( err < 0) TEST_ERROR; err = H5Fclose(fid1); if( err < 0) TEST_ERROR; PASSED(); return 0; error: H5E_BEGIN_TRY { H5Pclose(plist_id); H5Dclose(dset_id); H5Tclose(part_t); H5PTclose(table); H5Fclose(fid1); } H5E_END_TRY H5_FAILED(); return -1; }
int main(int argc, char ** argv) { if(argc < 2){ std::cout << "please specify a file" << std::endl; return 0; } std::string filename = argv[1]; hsize_t offset[2] = {2,3}; if (argc == 4){ offset[0] = atoi(argv[2]); offset[1] = atoi(argv[3]); } int m_rows = 4; int m_cols = 4; herr_t status; hid_t file = H5Fopen (filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); hid_t dset = H5Dopen (file, "DATASET", H5P_DEFAULT); hid_t dcpl = H5Dget_create_plist (dset); H5D_layout_t layout = H5Pget_layout (dcpl); hid_t space = H5Dget_space (dset); hsize_t dims[2], mdims[2]; status = H5Sget_simple_extent_dims(space,dims,mdims); int dim1 = dims[0]; int dim2 = dims[1]; const int length = m_rows*m_cols; double * data = new double[length]; // Select from the file hsize_t count[2] = {4,4}; status = H5Sselect_hyperslab (space, H5S_SELECT_SET, offset, NULL, count, NULL); // Create memory space hsize_t dim_out[2] = {m_rows, m_cols}; hsize_t offset_out[2] = {0,0}; hsize_t count_out[2] = {m_rows, m_cols}; hid_t memspace = H5Screate_simple(2,dim_out,NULL); status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); status = H5Dread (dset, H5T_NATIVE_DOUBLE, memspace, space, H5P_DEFAULT, data); // Print to screen for(int r=0; r<m_rows; ++r){ for(int c=0; c<m_cols; ++c){ std::cout << std::setfill (' ') << std::setw (8); std::cout << data[c + r*m_cols]; } std::cout << std::endl; } delete [] data; status = H5Pclose (dcpl); status = H5Dclose (dset); status = H5Sclose (space); status = H5Fclose (file); return 0; }
static herr_t group_info (hid_t loc_id, const char *name, const H5L_info_t *linfo, void *opdata) { hid_t datasetID; /* dataset identifier */ hid_t datatypeID; /* datatype identifier */ hid_t propertyID; /* data_property identifier */ H5T_class_t t_class; hsize_t chunk_dims_out[2]; int rank_chunk; /* avoid warnings */ opdata = opdata; linfo = linfo; /* * Open the datasets using their names. */ datasetID = H5Dopen (loc_id, name, H5P_DEFAULT); /* * Display dataset name. */ printf("\nName : %s\n", name); /* * Display dataset information. */ datatypeID = H5Dget_type(datasetID); /* get datatype*/ propertyID = H5Dget_create_plist(datasetID); /* get creation property list */ /* * Check if dataset is chunked. */ if(H5D_CHUNKED == H5Pget_layout(propertyID)) { /* * get chunking information: rank and dimensions. */ rank_chunk = H5Pget_chunk(propertyID, 2, chunk_dims_out); printf("chunk rank %d, dimensions %lu x %lu\n", rank_chunk, (unsigned long)(chunk_dims_out[0]), (unsigned long)(chunk_dims_out[1])); } else { t_class = H5Tget_class(datatypeID); if(t_class < 0) { puts(" Invalid datatype.\n"); } else { if(t_class == H5T_INTEGER) puts(" Datatype is 'H5T_NATIVE_INTEGER'.\n"); if(t_class == H5T_FLOAT) puts(" Datatype is 'H5T_NATIVE_FLOAT'.\n"); if(t_class == H5T_STRING) puts(" Datatype is 'H5T_NATIVE_STRING'.\n"); if(t_class == H5T_BITFIELD) puts(" Datatype is 'H5T_NATIVE_BITFIELD'.\n"); if(t_class == H5T_OPAQUE) puts(" Datatype is 'H5T_NATIVE_OPAQUE'.\n"); if(t_class == H5T_COMPOUND) puts(" Datatype is 'H5T_NATIVE_COMPOUND'.\n"); } } H5Dclose(datasetID); H5Pclose(propertyID); H5Tclose(datatypeID); return 0; }
/*------------------------------------------------------------------------- * Function: diff_datasetid * * Purpose: check for comparable datasets and read into a compatible * memory type * * Return: Number of differences found * * Programmer: Pedro Vicente, [email protected] * * Date: May 9, 2003 * * Modifications: * * * October 2006: Read by hyperslabs for big datasets. * * A threshold of H5TOOLS_MALLOCSIZE (128 MB) is the limit upon which I/O hyperslab is done * i.e., if the memory needed to read a dataset is greater than this limit, * then hyperslab I/O is done instead of one operation I/O * For each dataset, the memory needed is calculated according to * * memory needed = number of elements * size of each element * * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations * are done * * H5Dread( input_dataset1 ) * H5Dread( input_dataset2 ) * * with all elements in the datasets selected. If the memory needed is greater than * H5TOOLS_MALLOCSIZE, then the following operations are done instead: * * a strip mine is defined for each dimension k (a strip mine is defined as a * hyperslab whose size is memory manageable) according to the formula * * (1) strip_mine_size[k ] = MIN(dimension[k ], H5TOOLS_BUFSIZE / size of memory type) * * where H5TOOLS_BUFSIZE is a constant currently defined as 1MB. This formula assures * that for small datasets (small relative to the H5TOOLS_BUFSIZE constant), the strip * mine size k is simply defined as its dimension k, but for larger datasets the * hyperslab size is still memory manageable. * a cycle is done until the number of elements in the dataset is reached. In each * iteration, two parameters are defined for the function H5Sselect_hyperslab, * the start and size of each hyperslab, according to * * (2) hyperslab_size [k] = MIN(dimension[k] - hyperslab_offset[k], strip_mine_size [k]) * * where hyperslab_offset [k] is initially set to zero, and later incremented in * hyperslab_size[k] offsets. The reason for the operation * * dimension[k] - hyperslab_offset[k] * * in (2) is that, when using the strip mine size, it assures that the "remaining" part * of the dataset that does not fill an entire strip mine is processed. * *------------------------------------------------------------------------- */ hsize_t diff_datasetid( hid_t did1, hid_t did2, const char *obj1_name, const char *obj2_name, diff_opt_t *options) { hid_t sid1=-1; hid_t sid2=-1; hid_t f_tid1=-1; hid_t f_tid2=-1; hid_t m_tid1=-1; hid_t m_tid2=-1; hid_t dcpl1 = -1; hid_t dcpl2 = -1; H5D_layout_t stl1 = -1; H5D_layout_t stl2 = -1; size_t m_size1; size_t m_size2; H5T_sign_t sign1; H5T_sign_t sign2; int rank1; int rank2; hsize_t nelmts1; hsize_t nelmts2; hsize_t dims1[H5S_MAX_RANK]; hsize_t dims2[H5S_MAX_RANK]; hsize_t maxdim1[H5S_MAX_RANK]; hsize_t maxdim2[H5S_MAX_RANK]; const char *name1=NULL; /* relative names */ const char *name2=NULL; hsize_t storage_size1; hsize_t storage_size2; hsize_t nfound=0; /* number of differences found */ int can_compare=1; /* do diff or not */ void *buf1=NULL; void *buf2=NULL; void *sm_buf1=NULL; void *sm_buf2=NULL; hid_t sm_space; /*stripmine data space */ size_t need; /* bytes needed for malloc */ int i; unsigned int vl_data = 0; /*contains VL datatypes */ h5difftrace("diff_datasetid start\n"); /* Get the dataspace handle */ if ( (sid1 = H5Dget_space(did1)) < 0 ) goto error; /* Get rank */ if ( (rank1 = H5Sget_simple_extent_ndims(sid1)) < 0 ) goto error; /* Get the dataspace handle */ if ( (sid2 = H5Dget_space(did2)) < 0 ) goto error; /* Get rank */ if ( (rank2 = H5Sget_simple_extent_ndims(sid2)) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid1,dims1,maxdim1) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid2,dims2,maxdim2) < 0 ) { goto error; } /*------------------------------------------------------------------------- * get the file data type *------------------------------------------------------------------------- */ /* Get the data type */ if ( (f_tid1 = H5Dget_type(did1)) < 0 ) goto error; /* Get the data type */ if ( (f_tid2 = H5Dget_type(did2)) < 0 ) { goto error; } /*------------------------------------------------------------------------- * get the storage layout type *------------------------------------------------------------------------- */ if((dcpl1 = H5Dget_create_plist(did1)) < 0) goto error; if((dcpl2 = H5Dget_create_plist(did2)) < 0) goto error; if((stl1 = H5Pget_layout(dcpl1)) < 0) goto error; if((stl2 = H5Pget_layout(dcpl2)) < 0) goto error; /*------------------------------------------------------------------------- * check for empty datasets *------------------------------------------------------------------------- */ h5difftrace("check for empty datasets\n"); storage_size1=H5Dget_storage_size(did1); storage_size2=H5Dget_storage_size(did2); if (storage_size1==0 || storage_size2==0) { if (stl1==H5D_VIRTUAL || stl2==H5D_VIRTUAL) { if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) parallel_print("Warning: <%s> or <%s> is a virtual dataset\n", obj1_name, obj2_name); } else { if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) parallel_print("Not comparable: <%s> or <%s> is an empty dataset\n", obj1_name, obj2_name); can_compare=0; options->not_cmp=1; } } /*------------------------------------------------------------------------- * check for comparable TYPE and SPACE *------------------------------------------------------------------------- */ if (diff_can_type(f_tid1, f_tid2, rank1, rank2, dims1, dims2, maxdim1, maxdim2, obj1_name, obj2_name, options, 0)!=1) { can_compare=0; } /*------------------------------------------------------------------------- * memory type and sizes *------------------------------------------------------------------------- */ h5difftrace("check for memory type and sizes\n"); if ((m_tid1=h5tools_get_native_type(f_tid1)) < 0) goto error; if ((m_tid2=h5tools_get_native_type(f_tid2)) < 0) goto error; m_size1 = H5Tget_size( m_tid1 ); m_size2 = H5Tget_size( m_tid2 ); /*------------------------------------------------------------------------- * check for different signed/unsigned types *------------------------------------------------------------------------- */ if (can_compare) { h5difftrace("can_compare for sign\n"); sign1=H5Tget_sign(m_tid1); sign2=H5Tget_sign(m_tid2); if ( sign1 != sign2 ) { h5difftrace("sign1 != sign2\n"); if ((options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) { parallel_print("Not comparable: <%s> has sign %s ", obj1_name, get_sign(sign1)); parallel_print("and <%s> has sign %s\n", obj2_name, get_sign(sign2)); } can_compare=0; options->not_cmp=1; } } /* Check if type is either VLEN-data or VLEN-string to reclaim any * VLEN memory buffer later */ if( TRUE == h5tools_detect_vlen(m_tid1) ) vl_data = TRUE; /*------------------------------------------------------------------------ * only attempt to compare if possible *------------------------------------------------------------------------- */ if(can_compare) /* it is possible to compare */ { h5difftrace("can_compare attempt\n"); /*----------------------------------------------------------------- * get number of elements *------------------------------------------------------------------ */ nelmts1 = 1; for(i = 0; i < rank1; i++) nelmts1 *= dims1[i]; nelmts2 = 1; for(i = 0; i < rank2; i++) nelmts2 *= dims2[i]; HDassert(nelmts1 == nelmts2); /*----------------------------------------------------------------- * "upgrade" the smaller memory size *------------------------------------------------------------------ */ h5difftrace("upgrade the smaller memory size?\n"); if (FAIL == match_up_memsize (f_tid1, f_tid2, &m_tid1, &m_tid2, &m_size1, &m_size2)) goto error; /* print names */ if(obj1_name) name1 = diff_basename(obj1_name); if(obj2_name) name2 = diff_basename(obj2_name); /*---------------------------------------------------------------- * read/compare *----------------------------------------------------------------- */ need = (size_t)(nelmts1 * m_size1); /* bytes needed */ if(need < H5TOOLS_MALLOCSIZE) { buf1 = HDmalloc(need); buf2 = HDmalloc(need); } /* end if */ if(buf1 != NULL && buf2 != NULL) { h5difftrace("buf1 != NULL && buf2 != NULL\n"); if(H5Dread(did1, m_tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf1) < 0) goto error; if(H5Dread(did2, m_tid2, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf2) < 0) goto error; /* array diff */ nfound = diff_array(buf1, buf2, nelmts1, (hsize_t)0, rank1, dims1, options, name1, name2, m_tid1, did1, did2); /* reclaim any VL memory, if necessary */ if(vl_data) { H5Dvlen_reclaim(m_tid1, sid1, H5P_DEFAULT, buf1); H5Dvlen_reclaim(m_tid2, sid2, H5P_DEFAULT, buf2); } /* end if */ } /* end if */ else /* possibly not enough memory, read/compare by hyperslabs */ { size_t p_type_nbytes = m_size1; /*size of memory type */ hsize_t p_nelmts = nelmts1; /*total selected elmts */ hsize_t elmtno; /*counter */ int carry; /*counter carry value */ /* stripmine info */ hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ hsize_t sm_nbytes; /*bytes per stripmine */ hsize_t sm_nelmts; /*elements per stripmine*/ /* hyperslab info */ hsize_t hs_offset[H5S_MAX_RANK]; /*starting offset */ hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ hsize_t hs_nelmts; /*elements in request */ hsize_t zero[8]; /*vector of zeros */ /* * determine the strip mine size and allocate a buffer. The strip mine is * a hyperslab whose size is manageable. */ sm_nbytes = p_type_nbytes; for(i = rank1; i > 0; --i) { hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; if(size == 0) /* datum size > H5TOOLS_BUFSIZE */ size = 1; sm_size[i - 1] = MIN(dims1[i - 1], size); sm_nbytes *= sm_size[i - 1]; HDassert(sm_nbytes > 0); } /* end for */ /* malloc return code should be verified. * If fail, need to handle the error. * This else branch should be recoded as a separate function. * Note that there are many "goto error" within this branch * that fails to address freeing other objects created here. * E.g., sm_space. */ sm_buf1 = HDmalloc((size_t)sm_nbytes); HDassert(sm_buf1); sm_buf2 = HDmalloc((size_t)sm_nbytes); HDassert(sm_buf2); sm_nelmts = sm_nbytes / p_type_nbytes; sm_space = H5Screate_simple(1, &sm_nelmts, NULL); /* the stripmine loop */ HDmemset(hs_offset, 0, sizeof hs_offset); HDmemset(zero, 0, sizeof zero); for(elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) { /* calculate the hyperslab size */ if(rank1 > 0) { for(i = 0, hs_nelmts = 1; i < rank1; i++) { hs_size[i] = MIN(dims1[i] - hs_offset[i], sm_size[i]); hs_nelmts *= hs_size[i]; } /* end for */ if(H5Sselect_hyperslab(sid1, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if(H5Sselect_hyperslab(sid2, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if(H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL) < 0) goto error; } /* end if */ else hs_nelmts = 1; if(H5Dread(did1,m_tid1,sm_space,sid1,H5P_DEFAULT,sm_buf1) < 0) goto error; if(H5Dread(did2,m_tid2,sm_space,sid2,H5P_DEFAULT,sm_buf2) < 0) goto error; /* get array differences. in the case of hyperslab read, increment the number of differences found in each hyperslab and pass the position at the beggining for printing */ nfound += diff_array(sm_buf1, sm_buf2, hs_nelmts, elmtno, rank1, dims1, options, name1, name2, m_tid1, did1, did2); /* reclaim any VL memory, if necessary */ if(vl_data) { H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1); H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2); } /* end if */ /* calculate the next hyperslab offset */ for(i = rank1, carry = 1; i > 0 && carry; --i) { hs_offset[i - 1] += hs_size[i - 1]; if(hs_offset[i - 1] == dims1[i - 1]) hs_offset[i - 1] = 0; else carry = 0; } /* i */ } /* elmtno */ H5Sclose(sm_space); } /* hyperslab read */ } /*can_compare*/ /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ h5difftrace("compare attributes?\n"); /* free */ if(buf1 != NULL) { HDfree(buf1); buf1 = NULL; } /* end if */ if(buf2 != NULL) { HDfree(buf2); buf2 = NULL; } /* end if */ if(sm_buf1 != NULL) { HDfree(sm_buf1); sm_buf1 = NULL; } /* end if */ if(sm_buf2 != NULL) { HDfree(sm_buf2); sm_buf2 = NULL; } /* end if */ H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); } H5E_END_TRY; h5difftrace("diff_datasetid finish\n"); return nfound; error: options->err_stat=1; /* free */ if (buf1!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid1, sid1, H5P_DEFAULT, buf1); HDfree(buf1); buf1=NULL; } if (buf2!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid2, sid2, H5P_DEFAULT, buf2); HDfree(buf2); buf2=NULL; } if (sm_buf1!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1); HDfree(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { /* reclaim any VL memory, if necessary */ if(vl_data) H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2); HDfree(sm_buf2); sm_buf2=NULL; } /* disable error reporting */ H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); /* enable error reporting */ } H5E_END_TRY; h5difftrace("diff_datasetid errored\n"); return nfound; }
int h5repack_cmp_pl(const char *fname1, const char *fname2) { int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */ hid_t fid1=-1; /* file ID */ hid_t fid2=-1; /* file ID */ hid_t dset1=-1; /* dataset ID */ hid_t dset2=-1; /* dataset ID */ hid_t gid=-1; /* group ID */ hid_t dcpl1=-1; /* dataset creation property list ID */ hid_t dcpl2=-1; /* dataset creation property list ID */ hid_t gcplid=-1; /* group creation property list */ unsigned crt_order_flag1; /* group creation order flag */ unsigned crt_order_flag2; /* group creation order flag */ trav_table_t *trav=NULL; int ret=1; unsigned int i; /*------------------------------------------------------------------------- * open the files *------------------------------------------------------------------------- */ /* disable error reporting */ H5E_BEGIN_TRY { /* Open the files */ if ((fid1 = H5Fopen(fname1,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) { error_msg("<%s>: %s\n", fname1, H5FOPENERROR ); return -1; } if ((fid2 = H5Fopen(fname2,H5F_ACC_RDONLY,H5P_DEFAULT)) < 0 ) { error_msg("<%s>: %s\n", fname2, H5FOPENERROR ); H5Fclose(fid1); return -1; } /* enable error reporting */ } H5E_END_TRY; /*------------------------------------------------------------------------- * get file table list of objects *------------------------------------------------------------------------- */ trav_table_init(&trav); if(h5trav_gettable(fid1, trav) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed"); /*------------------------------------------------------------------------- * traverse the suppplied object list *------------------------------------------------------------------------- */ for(i = 0; i < trav->nobjs; i++) { if(trav->objs[i].type == H5TRAV_TYPE_GROUP) { if ((gid = H5Gopen2(fid1, trav->objs[i].name, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); if ((gcplid = H5Gget_create_plist(gid)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_create_plist failed"); if (H5Pget_link_creation_order(gcplid, &crt_order_flag1) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_link_creation_order failed"); if (H5Pclose(gcplid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); if (H5Gclose(gid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); if ((gid = H5Gopen2(fid2, trav->objs[i].name, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed"); if ((gcplid = H5Gget_create_plist(gid)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_create_plist failed"); if (H5Pget_link_creation_order(gcplid, &crt_order_flag2) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_link_creation_order failed"); if (H5Pclose(gcplid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); if (H5Gclose(gid) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gclose failed"); if (crt_order_flag1 != crt_order_flag2) { error_msg("property lists for <%s> are different\n",trav->objs[i].name); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "property lists failed"); } } else if(trav->objs[i].type == H5TRAV_TYPE_DATASET) { if((dset1 = H5Dopen2(fid1, trav->objs[i].name, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); if((dset2 = H5Dopen2(fid2, trav->objs[i].name, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed"); if((dcpl1 = H5Dget_create_plist(dset1)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); if((dcpl2 = H5Dget_create_plist(dset2)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed"); /*------------------------------------------------------------------------- * compare the property lists *------------------------------------------------------------------------- */ if((ret = H5Pequal(dcpl1, dcpl2)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pequal failed"); if(ret == 0) { error_msg("property lists for <%s> are different\n",trav->objs[i].name); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "property lists failed"); } /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ if(H5Pclose(dcpl1) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); if(H5Pclose(dcpl2) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); if(H5Dclose(dset1) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); if(H5Dclose(dset2) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); } /*if*/ } /*i*/ /*------------------------------------------------------------------------- * free *------------------------------------------------------------------------- */ trav_table_free(trav); /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ H5Fclose(fid1); H5Fclose(fid2); return ret; /*------------------------------------------------------------------------- * error *------------------------------------------------------------------------- */ done: H5E_BEGIN_TRY { H5Pclose(dcpl1); H5Pclose(dcpl2); H5Dclose(dset1); H5Dclose(dset2); H5Fclose(fid1); H5Fclose(fid2); H5Pclose(gcplid); H5Gclose(gid); trav_table_free(trav); } H5E_END_TRY; return ret_value; }
DatasetCreationProperties Dataset::getCopyOfCreationProperties() const { // {{{ return DatasetCreationProperties(assertSuccess( "getting dataset creation properties", H5Dget_create_plist(getId()) )); } // }}}
int do_copy_refobjs(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *options) /* repack options */ { hid_t grp_in=(-1); /* read group ID */ hid_t grp_out=(-1); /* write group ID */ hid_t dset_in=(-1); /* read dataset ID */ hid_t dset_out=(-1); /* write dataset ID */ hid_t type_in=(-1); /* named type ID */ hid_t dcpl_id=(-1); /* dataset creation property list ID */ hid_t space_id=(-1); /* space ID */ hid_t ftype_id=(-1); /* file data type ID */ hid_t mtype_id=(-1); /* memory data type ID */ size_t msize; /* memory size of memory type */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ unsigned int i, j; int k; /*------------------------------------------------------------------------- * browse *------------------------------------------------------------------------- */ for ( i = 0; i < travt->nobjs; i++) { switch ( travt->objs[i].type ) { /*------------------------------------------------------------------------- * H5G_GROUP *------------------------------------------------------------------------- */ case H5G_GROUP: /*------------------------------------------------------------------------- * copy referenced objects in attributes *------------------------------------------------------------------------- */ if ((grp_out=H5Gopen(fidout,travt->objs[i].name))<0) goto error; if((grp_in = H5Gopen (fidin,travt->objs[i].name))<0) goto error; if (copy_refs_attr(grp_in,grp_out,options,travt,fidout)<0) goto error; if (H5Gclose(grp_out)<0) goto error; if (H5Gclose(grp_in)<0) goto error; /*------------------------------------------------------------------------- * check for hard links *------------------------------------------------------------------------- */ if (travt->objs[i].nlinks) { for ( j=0; j<travt->objs[i].nlinks; j++) { H5Glink(fidout, H5G_LINK_HARD, travt->objs[i].name, travt->objs[i].links[j].new_name); } } break; /*------------------------------------------------------------------------- * H5G_DATASET *------------------------------------------------------------------------- */ case H5G_DATASET: if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) goto error; if ((space_id=H5Dget_space(dset_in))<0) goto error; if ((ftype_id=H5Dget_type (dset_in))<0) goto error; if ((dcpl_id=H5Dget_create_plist(dset_in))<0) goto error; if ( (rank=H5Sget_simple_extent_ndims(space_id))<0) goto error; if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0) goto error; nelmts=1; for (k=0; k<rank; k++) nelmts*=dims[k]; if ((mtype_id=h5tools_get_native_type(ftype_id))<0) goto error; if ((msize=H5Tget_size(mtype_id))==0) goto error; /*------------------------------------------------------------------------- * check if the dataset creation property list has filters that * are not registered in the current configuration * 1) the external filters GZIP and SZIP might not be available * 2) the internal filters might be turned off *------------------------------------------------------------------------- */ if (h5tools_canreadf((NULL),dcpl_id)==1) { /*------------------------------------------------------------------------- * test for a valid output dataset *------------------------------------------------------------------------- */ dset_out = FAIL; /*------------------------------------------------------------------------- * object references are a special case * we cannot just copy the buffers, but instead we recreate the reference *------------------------------------------------------------------------- */ if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) { H5G_obj_t1 obj_type; hid_t refobj_id; hobj_ref_t *refbuf=NULL; /* buffer for object references */ hobj_ref_t *buf=NULL; const char* refname; unsigned u; /*------------------------------------------------------------------------- * read to memory *------------------------------------------------------------------------- */ if (nelmts) { buf=(void *) HDmalloc((unsigned)(nelmts*msize)); if ( buf==NULL){ printf( "cannot read into memory\n" ); goto error; } if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(dset_in,H5R_OBJECT,buf))<0) goto error; refbuf=HDcalloc((unsigned)nelmts,msize); if ( refbuf==NULL){ printf( "cannot allocate memory\n" ); goto error; } for ( u=0; u<nelmts; u++) { H5E_BEGIN_TRY { if ((refobj_id = H5Rdereference(dset_in,H5R_OBJECT,&buf[u]))<0) continue; } H5E_END_TRY; /* get the name. a valid name could only occur in the second traversal of the file */ if ((refname=MapIdToName(refobj_id,travt))!=NULL) { /* create the reference, -1 parameter for objects */ if (H5Rcreate(&refbuf[u],fidout,refname,H5R_OBJECT,-1)<0) goto error; if(options->verbose) { printf(FORMAT_OBJ,"dset",travt->objs[i].name ); printf("object <%s> object reference created to <%s>\n", travt->objs[i].name, refname); } }/*refname*/ close_obj(obj_type,refobj_id); }/* u */ }/*nelmts*/ /*------------------------------------------------------------------------- * create/write dataset/close *------------------------------------------------------------------------- */ if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) goto error; if (nelmts) { if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) goto error; } if (buf) free(buf); if (refbuf) free(refbuf); }/*H5T_STD_REF_OBJ*/ /*------------------------------------------------------------------------- * dataset region references *------------------------------------------------------------------------- */ else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) { H5G_obj_t1 obj_type; hid_t refobj_id; hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */ hdset_reg_ref_t *buf=NULL; /* output buffer */ const char* refname; unsigned u; /*------------------------------------------------------------------------- * read input to memory *------------------------------------------------------------------------- */ if (nelmts) { buf=(void *) HDmalloc((unsigned)(nelmts*msize)); if ( buf==NULL){ printf( "cannot read into memory\n" ); goto error; } if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(dset_in,H5R_DATASET_REGION,buf))<0) goto error; /*------------------------------------------------------------------------- * create output *------------------------------------------------------------------------- */ refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */ if ( refbuf==NULL){ printf( "cannot allocate memory\n" ); goto error; } for ( u=0; u<nelmts; u++) { H5E_BEGIN_TRY { if ((refobj_id = H5Rdereference(dset_in,H5R_DATASET_REGION,&buf[u]))<0) continue; } H5E_END_TRY; /* get the name. a valid name could only occur in the second traversal of the file */ if ((refname=MapIdToName(refobj_id,travt))!=NULL) { hid_t region_id; /* region id of the referenced dataset */ if ((region_id = H5Rget_region(dset_in,H5R_DATASET_REGION,&buf[u]))<0) goto error; /* create the reference, we need the space_id */ if (H5Rcreate(&refbuf[u],fidout,refname,H5R_DATASET_REGION,region_id)<0) goto error; if (H5Sclose(region_id)<0) goto error; if(options->verbose) { printf(FORMAT_OBJ,"dset",travt->objs[i].name ); printf("object <%s> region reference created to <%s>\n", travt->objs[i].name, refname); } }/*refname*/ close_obj(obj_type,refobj_id); }/* u */ }/*nelmts*/ /*------------------------------------------------------------------------- * create/write dataset/close *------------------------------------------------------------------------- */ if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) goto error; if (nelmts) { if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) goto error; } if (buf) free(buf); if (refbuf) free(refbuf); } /* H5T_STD_REF_DSETREG */ /*------------------------------------------------------------------------- * not references, open previously created object in 1st traversal *------------------------------------------------------------------------- */ else { if ((dset_out=H5Dopen(fidout,travt->objs[i].name))<0) goto error; } assert(dset_out!=FAIL); /*------------------------------------------------------------------------- * copy referenced objects in attributes *------------------------------------------------------------------------- */ if (copy_refs_attr(dset_in,dset_out,options,travt,fidout)<0) goto error; /*------------------------------------------------------------------------- * check for hard links *------------------------------------------------------------------------- */ if (travt->objs[i].nlinks) { for ( j=0; j<travt->objs[i].nlinks; j++){ H5Glink(fidout, H5G_LINK_HARD, travt->objs[i].name, travt->objs[i].links[j].new_name); } } if (H5Dclose(dset_out)<0) goto error; }/*can_read*/
/*------------------------------------------------------------------------- * Function: diff_dataset * * Purpose: check for comparable datasets and read into a compatible * memory type * * Return: Number of differences found * * Programmer: Pedro Vicente, [email protected] * * Date: May 9, 2003 * *------------------------------------------------------------------------- */ hsize_t diff_dataset( hid_t file1_id, hid_t file2_id, const char *obj1_name, const char *obj2_name, diff_opt_t *options) { hid_t did1 = -1; hid_t did2 = -1; hid_t dcpl1 = -1; hid_t dcpl2 = -1; hsize_t nfound = 0; /*------------------------------------------------------------------------- * open the handles *------------------------------------------------------------------------- */ /* disable error reporting */ H5E_BEGIN_TRY { /* Open the datasets */ if((did1 = H5Dopen2(file1_id, obj1_name, H5P_DEFAULT)) < 0) { parallel_print("Cannot open dataset <%s>\n", obj1_name); goto error; } if((did2 = H5Dopen2(file2_id, obj2_name, H5P_DEFAULT)) < 0) { parallel_print("Cannot open dataset <%s>\n", obj2_name); goto error; } /* enable error reporting */ } H5E_END_TRY; if((dcpl1 = H5Dget_create_plist(did1)) < 0) goto error; if((dcpl2 = H5Dget_create_plist(did2)) < 0) { goto error; } /*------------------------------------------------------------------------- * check if the dataset creation property list has filters that * are not registered in the current configuration * 1) the external filters GZIP and SZIP might not be available * 2) the internal filters might be turned off *------------------------------------------------------------------------- */ if ((h5tools_canreadf((options->m_verbose?obj1_name:NULL),dcpl1)==1) && (h5tools_canreadf((options->m_verbose?obj2_name:NULL),dcpl2)==1)) { nfound=diff_datasetid(did1, did2, obj1_name, obj2_name, options); } /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ /* disable error reporting */ H5E_BEGIN_TRY { H5Pclose(dcpl1); H5Pclose(dcpl2); H5Dclose(did1); H5Dclose(did2); /* enable error reporting */ } H5E_END_TRY; return nfound; error: options->err_stat=1; /* disable error reporting */ H5E_BEGIN_TRY { H5Pclose(dcpl1); H5Pclose(dcpl2); H5Dclose(did1); H5Dclose(did2); /* enable error reporting */ } H5E_END_TRY; return nfound; }