/* * test_objnames * Tests that UTF-8 can be used for object names in the file. * Tests groups, datasets, named datatypes, and soft links. * Note that this test doesn't actually mark the names as being * in UTF-8. At the time this test was written, that feature * didn't exist in HDF5, and when the character encoding property * was added to links it didn't change how they were stored in the file, * -JML 2/2/2006 */ void test_objnames(hid_t fid, const char* string) { hid_t grp_id, grp1_id, grp2_id, grp3_id; hid_t type_id, dset_id, space_id; char read_buf[MAX_STRING_LENGTH]; char path_buf[MAX_PATH_LENGTH]; hsize_t dims=1; hobj_ref_t obj_ref; herr_t ret; /* Create a group with a UTF-8 name */ grp_id = H5Gcreate2(fid, string, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(grp_id, FAIL, "H5Gcreate2"); /* Set a comment on the group to test that we can access the group * Also test that UTF-8 comments can be read. */ ret = H5Oset_comment_by_name(fid, string, string, H5P_DEFAULT); CHECK(ret, FAIL, "H5Oset_comment_by_name"); ret = H5Oget_comment_by_name(fid, string, read_buf, (size_t)MAX_STRING_LENGTH, H5P_DEFAULT); CHECK(ret, FAIL, "H5Oget_comment_by_name"); ret = H5Gclose(grp_id); CHECK(ret, FAIL, "H5Gclose"); VERIFY(HDstrcmp(string, read_buf), 0, "strcmp"); /* Create a new dataset with a UTF-8 name */ grp1_id = H5Gcreate2(fid, GROUP1_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(grp1_id, FAIL, "H5Gcreate2"); space_id = H5Screate_simple(RANK, &dims, NULL); CHECK(space_id, FAIL, "H5Screate_simple"); dset_id = H5Dcreate2(grp1_id, string, H5T_NATIVE_INT, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(dset_id, FAIL, "H5Dcreate2"); /* Make sure that dataset can be opened again */ ret = H5Dclose(dset_id); CHECK(ret, FAIL, "H5Dclose"); ret = H5Sclose(space_id); CHECK(ret, FAIL, "H5Sclose"); dset_id = H5Dopen2(grp1_id, string, H5P_DEFAULT); CHECK(ret, FAIL, "H5Dopen2"); ret = H5Dclose(dset_id); CHECK(ret, FAIL, "H5Dclose"); ret = H5Gclose(grp1_id); CHECK(ret, FAIL, "H5Gclose"); /* Do the same for a named datatype */ grp2_id = H5Gcreate2(fid, GROUP2_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(grp2_id, FAIL, "H5Gcreate2"); type_id = H5Tcreate(H5T_OPAQUE, (size_t)1); CHECK(type_id, FAIL, "H5Tcreate"); ret = H5Tcommit2(grp2_id, string, type_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(type_id, FAIL, "H5Tcommit2"); ret = H5Tclose(type_id); CHECK(type_id, FAIL, "H5Tclose"); type_id = H5Topen2(grp2_id, string, H5P_DEFAULT); CHECK(type_id, FAIL, "H5Topen2"); ret = H5Tclose(type_id); CHECK(type_id, FAIL, "H5Tclose"); /* Don't close the group -- use it to test that object references * can refer to objects named in UTF-8 */ space_id = H5Screate_simple(RANK, &dims, NULL); CHECK(space_id, FAIL, "H5Screate_simple"); dset_id = H5Dcreate2(grp2_id, DSET3_NAME, H5T_STD_REF_OBJ, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(ret, FAIL, "H5Dcreate2"); /* Create reference to named datatype */ ret = H5Rcreate(&obj_ref, grp2_id, string, H5R_OBJECT, -1); CHECK(ret, FAIL, "H5Rcreate"); /* Write selection and read it back*/ ret = H5Dwrite(dset_id, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, &obj_ref); CHECK(ret, FAIL, "H5Dwrite"); ret = H5Dread(dset_id, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, &obj_ref); CHECK(ret, FAIL, "H5Dread"); /* Ensure that we can open named datatype using object reference */ type_id = H5Rdereference2(dset_id, H5P_DEFAULT, H5R_OBJECT, &obj_ref); CHECK(type_id, FAIL, "H5Rdereference2"); ret = H5Tcommitted(type_id); VERIFY(ret, 1, "H5Tcommitted"); ret = H5Tclose(type_id); CHECK(type_id, FAIL, "H5Tclose"); ret = H5Dclose(dset_id); CHECK(ret, FAIL, "H5Dclose"); ret = H5Sclose(space_id); CHECK(ret, FAIL, "H5Sclose"); ret = H5Gclose(grp2_id); CHECK(ret, FAIL, "H5Gclose"); /* Create "group3". Build a hard link from group3 to group2, which has * a datatype with the UTF-8 name. Create a soft link in group3 * pointing through the hard link to the datatype. Give the soft * link a name in UTF-8. Ensure that the soft link works. */ grp3_id = H5Gcreate2(fid, GROUP3_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(grp3_id, FAIL, "H5Gcreate2"); ret = H5Lcreate_hard(fid, GROUP2_NAME, grp3_id, GROUP2_NAME, H5P_DEFAULT, H5P_DEFAULT); CHECK(ret, FAIL, "H5Lcreate_hard"); HDstrcpy(path_buf, GROUP2_NAME); HDstrcat(path_buf, "/"); HDstrcat(path_buf, string); ret = H5Lcreate_hard(grp3_id, path_buf, H5L_SAME_LOC, string, H5P_DEFAULT, H5P_DEFAULT); CHECK(ret, FAIL, "H5Lcreate_hard"); /* Open named datatype using soft link */ type_id = H5Topen2(grp3_id, string, H5P_DEFAULT); CHECK(type_id, FAIL, "H5Topen2"); ret = H5Tclose(type_id); CHECK(type_id, FAIL, "H5Tclose"); ret = H5Gclose(grp3_id); CHECK(ret, FAIL, "H5Gclose"); }
/*------------------------------------------------------------------------- * Function: copy_attr * * Purpose: copy attributes located in LOC_IN, which is obtained either from * loc_id = H5Gopen2( fid, name); * loc_id = H5Dopen2( fid, name); * loc_id = H5Topen2( fid, name); * * Return: 0, ok, -1 no *------------------------------------------------------------------------- */ int copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options) { int ret_value = 0; hid_t attr_id = -1; /* attr ID */ hid_t attr_out = -1; /* attr ID */ hid_t space_id = -1; /* space ID */ hid_t ftype_id = -1; /* file type ID */ hid_t wtype_id = -1; /* read/write type ID */ size_t msize; /* size of type */ void *buf = NULL; /* data buffer */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ htri_t is_named; /* Whether the datatype is named */ hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ char name[255]; H5O_info_t oinfo; /* object info */ int j; unsigned u; hbool_t is_ref = 0; H5T_class_t type_class = -1; if (H5Oget_info(loc_in, &oinfo) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Oget_info failed"); /*------------------------------------------------------------------------- * copy all attributes *------------------------------------------------------------------------- */ for (u = 0; u < (unsigned) oinfo.num_attrs; u++) { /* open attribute */ if ((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) u, H5P_DEFAULT, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aopen_by_idx failed"); /* get name */ if (H5Aget_name(attr_id, (size_t) 255, name) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); /* get the file datatype */ if ((ftype_id = H5Aget_type(attr_id)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aget_type failed"); /* Check if the datatype is committed */ if ((is_named = H5Tcommitted(ftype_id)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tcommitted failed"); if (is_named && travt) { hid_t fidout = -1; /* Create out file id */ if ((fidout = H5Iget_file_id(loc_out)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Iget_file_id failed"); /* Copy named dt */ if ((wtype_id = copy_named_datatype(ftype_id, fidout, named_dt_head_p, travt, options)) < 0) { H5Fclose(fidout); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_named_datatype failed"); } /* end if */ if (H5Fclose(fidout) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fclose failed"); } /* end if */ else { if (options->use_native == 1) wtype_id = H5Tget_native_type(ftype_id, H5T_DIR_DEFAULT); else wtype_id = H5Tcopy(ftype_id); } /* end else */ /* get the dataspace handle */ if ((space_id = H5Aget_space(attr_id)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aget_space failed"); /* get dimensions */ if ((rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed"); nelmts = 1; for (j = 0; j < rank; j++) nelmts *= dims[j]; if ((msize = H5Tget_size(wtype_id)) == 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed"); /*------------------------------------------------------------------------- * object references are a special case. We cannot just copy the buffers, * but instead we recreate the reference. * This is done on a second sweep of the file that just copies the referenced * objects at copy_refs_attr() *------------------------------------------------------------------------- */ type_class = H5Tget_class(wtype_id); is_ref = (type_class == H5T_REFERENCE); if (type_class == H5T_VLEN || type_class == H5T_ARRAY) { hid_t base_type = -1; base_type = H5Tget_super(ftype_id); is_ref = (is_ref || (H5Tget_class(base_type) == H5T_REFERENCE)); if (H5Tclose(base_type) < 0) H5TOOLS_INFO(H5E_tools_min_id_g, "H5Tclose base_type failed"); } if (type_class == H5T_COMPOUND) { int nmembers = H5Tget_nmembers(wtype_id); for (j = 0; j < nmembers; j++) { hid_t mtid = H5Tget_member_type(wtype_id, (unsigned)j); H5T_class_t mtclass = H5Tget_class(mtid); if (H5Tclose(mtid) < 0) H5TOOLS_INFO(H5E_tools_min_id_g, "H5Tclose mtid failed"); if (mtclass == H5T_REFERENCE) { is_ref = 1; break; } } /* for (j=0; i<nmembers; j++) */ } /* if (type_class == H5T_COMPOUND) */ if (!is_ref) { /*------------------------------------------------------------------------- * read to memory *------------------------------------------------------------------------- */ buf = (void *)HDmalloc((size_t)(nelmts * msize)); if (buf == NULL) { HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); } /* end if */ if (H5Aread(attr_id, wtype_id, buf) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aread failed"); /*------------------------------------------------------------------------- * copy *------------------------------------------------------------------------- */ if ((attr_out = H5Acreate2(loc_out, name, wtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Acreate2 failed on ,%s>", name); if (H5Awrite(attr_out, wtype_id, buf) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Awrite failed"); /*close*/ if (H5Aclose(attr_out) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aclose failed"); /* Check if we have VL data and string in the attribute's datatype that must * be reclaimed */ if (TRUE == h5tools_detect_vlen(wtype_id)) H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf); HDfree(buf); buf = NULL; } /*H5T_REFERENCE*/ if (options->verbose) printf(FORMAT_OBJ_ATTR, "attr", name); /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ if (H5Sclose(space_id) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); space_id = -1; if (H5Tclose(wtype_id) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); wtype_id = -1; if (H5Tclose(ftype_id) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose failed"); ftype_id = -1; if (H5Aclose(attr_id) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aclose failed"); attr_id = -1; } /* for u */ done: H5E_BEGIN_TRY { if (buf) { /* Check if we have VL data and string in the attribute's datatype that must * be reclaimed */ if (TRUE == h5tools_detect_vlen(wtype_id)) H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf); /* Free buf */ HDfree(buf); } /* end if */ H5Aclose(attr_out); H5Sclose(space_id); H5Tclose(wtype_id); H5Tclose(ftype_id); H5Aclose(attr_id); } H5E_END_TRY; return ret_value; } /* end copy_attr() */
/*------------------------------------------------------------------------- * Function: dataset_stats * * Purpose: Gather statistics about the dataset * * Return: Success: 0 * * Failure: -1 * * Programmer: Quincey Koziol * Tuesday, August 16, 2005 * *------------------------------------------------------------------------- */ static herr_t dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) { unsigned bin; /* "bin" the number of objects falls in */ hid_t did; /* Dataset ID */ hid_t sid; /* Dataspace ID */ hid_t tid; /* Datatype ID */ hid_t dcpl; /* Dataset creation property list ID */ hsize_t dims[H5S_MAX_RANK];/* Dimensions of dataset */ H5D_layout_t lout; /* Layout of dataset */ unsigned type_found; /* Whether the dataset's datatype was */ /* already found */ int ndims; /* Number of dimensions of dataset */ hsize_t storage; /* Size of dataset storage */ unsigned u; /* Local index variable */ int num_ext; /* Number of external files for a dataset */ int nfltr; /* Number of filters for a dataset */ H5Z_filter_t fltr; /* Filter identifier */ herr_t ret; /* Gather statistics about this type of object */ iter->uniq_dsets++; /* Get object header information */ iter->dset_ohdr_info.total_size += oi->hdr.space.total; iter->dset_ohdr_info.free_size += oi->hdr.space.free; did = H5Dopen2(iter->fid, name, H5P_DEFAULT); HDassert(did > 0); /* Update dataset metadata info */ iter->datasets_index_storage_size += oi->meta_size.obj.index_size; iter->datasets_heap_storage_size += oi->meta_size.obj.heap_size; /* Update attribute metadata info */ ret = attribute_stats(iter, oi); HDassert(ret >= 0); /* Get storage info */ storage = H5Dget_storage_size(did); /* Gather layout statistics */ dcpl = H5Dget_create_plist(did); HDassert(dcpl > 0); lout = H5Pget_layout(dcpl); HDassert(lout >= 0); /* Object header's total size for H5D_COMPACT layout includes raw data size */ /* "storage" also includes H5D_COMPACT raw data size */ if(lout == H5D_COMPACT) iter->dset_ohdr_info.total_size -= storage; /* Track the layout type for dataset */ (iter->dset_layouts[lout])++; /* Get the number of external files for the dataset */ num_ext = H5Pget_external_count(dcpl); assert (num_ext >= 0); /* Accumulate raw data size accordingly */ if(num_ext) { iter->nexternal += (unsigned long)num_ext; iter->dset_external_storage_size += (unsigned long)storage; } else iter->dset_storage_size += storage; /* Gather dataspace statistics */ sid = H5Dget_space(did); HDassert(sid > 0); ndims = H5Sget_simple_extent_dims(sid, dims, NULL); HDassert(ndims >= 0); /* Check for larger rank of dataset */ if((unsigned)ndims > iter->max_dset_rank) iter->max_dset_rank = (unsigned)ndims; /* Track the number of datasets with each rank */ (iter->dset_rank_count[ndims])++; /* Only gather dim size statistics on 1-D datasets */ if(ndims == 1) { iter->max_dset_dims = dims[0]; if(dims[0] < SIZE_SMALL_DSETS) (iter->small_dset_dims[(size_t)dims[0]])++; /* Add dim count to proper bin */ bin = ceil_log10((unsigned long)dims[0]); if((bin + 1) > iter->dset_dim_nbins) { /* Allocate more storage for info about dataset's datatype */ iter->dset_dim_bins = (unsigned long *)HDrealloc(iter->dset_dim_bins, (bin + 1) * sizeof(unsigned long)); HDassert(iter->dset_dim_bins); /* Initialize counts for intermediate bins */ while(iter->dset_dim_nbins < bin) iter->dset_dim_bins[iter->dset_dim_nbins++] = 0; iter->dset_dim_nbins++; /* Initialize count for this bin */ iter->dset_dim_bins[bin] = 1; } /* end if */ else (iter->dset_dim_bins[bin])++; } /* end if */ ret = H5Sclose(sid); HDassert(ret >= 0); /* Gather datatype statistics */ tid = H5Dget_type(did); HDassert(tid > 0); type_found = FALSE; for(u = 0; u < iter->dset_ntypes; u++) if(H5Tequal(iter->dset_type_info[u].tid, tid) > 0) { type_found = TRUE; break; } /* end for */ if(type_found) (iter->dset_type_info[u].count)++; else { unsigned curr_ntype = iter->dset_ntypes; /* Increment # of datatypes seen for datasets */ iter->dset_ntypes++; /* Allocate more storage for info about dataset's datatype */ iter->dset_type_info = (dtype_info_t *)HDrealloc(iter->dset_type_info, iter->dset_ntypes * sizeof(dtype_info_t)); HDassert(iter->dset_type_info); /* Initialize information about datatype */ iter->dset_type_info[curr_ntype].tid = H5Tcopy(tid); HDassert(iter->dset_type_info[curr_ntype].tid > 0); iter->dset_type_info[curr_ntype].count = 1; iter->dset_type_info[curr_ntype].named = 0; /* Set index for later */ u = curr_ntype; } /* end else */ /* Check if the datatype is a named datatype */ if(H5Tcommitted(tid) > 0) (iter->dset_type_info[u].named)++; ret = H5Tclose(tid); HDassert(ret >= 0); /* Track different filters */ if((nfltr = H5Pget_nfilters(dcpl)) >= 0) { if(nfltr == 0) iter->dset_comptype[0]++; for(u = 0; u < (unsigned)nfltr; u++) { fltr = H5Pget_filter2(dcpl, u, 0, 0, 0, 0, 0, NULL); if(fltr >= 0) { if(fltr < (H5_NFILTERS_IMPL - 1)) iter->dset_comptype[fltr]++; else iter->dset_comptype[H5_NFILTERS_IMPL - 1]++; /*other filters*/ } /* end if */ } /* end for */ } /* endif nfltr */ ret = H5Pclose(dcpl); HDassert(ret >= 0); ret = H5Dclose(did); HDassert(ret >= 0); return 0; } /* end dataset_stats() */
/*------------------------------------------------------------------------- * Function: find_objs_cb * * Purpose: Callback to find objects, committed types and store them in tables * * Return: Success: SUCCEED * * Failure: FAIL * * Programmer: Ruey-Hsia Li * * Modifications: * *------------------------------------------------------------------------- */ static herr_t find_objs_cb(const char *name, const H5O_info_t *oinfo, const char *already_seen, void *op_data) { find_objs_t *info = (find_objs_t*)op_data; herr_t ret_value = 0; switch(oinfo->type) { case H5O_TYPE_GROUP: if(NULL == already_seen) add_obj(info->group_table, oinfo->addr, name, TRUE); break; case H5O_TYPE_DATASET: if(NULL == already_seen) { hid_t dset; /* Add the dataset to the list of objects */ add_obj(info->dset_table, oinfo->addr, name, TRUE); /* Check for a dataset that uses a named datatype */ if((dset = H5Dopen2(info->fid, name, H5P_DEFAULT)) >= 0) { hid_t type = H5Dget_type(dset); if(H5Tcommitted(type) > 0) { H5O_info_t type_oinfo; H5Oget_info(type, &type_oinfo); if(search_obj(info->type_table, type_oinfo.addr) == NULL) add_obj(info->type_table, type_oinfo.addr, name, FALSE); } /* end if */ H5Tclose(type); H5Dclose(dset); } /* end if */ else ret_value = FAIL; } /* end if */ break; case H5O_TYPE_NAMED_DATATYPE: if(NULL == already_seen) { obj_t *found_obj; if((found_obj = search_obj(info->type_table, oinfo->addr)) == NULL) add_obj(info->type_table, oinfo->addr, name, TRUE); else { /* Use latest version of name */ HDfree(found_obj->objname); found_obj->objname = HDstrdup(name); /* Mark named datatype as having valid name */ found_obj->recorded = TRUE; } /* end else */ } /* end if */ break; default: break; } /* end switch */ return ret_value; }