/* * Class: hdf_hdf5lib_H5 * Method: H5AreadVL * Signature: (JJJJJ[Ljava/lang/String;)I */ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5AreadVL (JNIEnv *env, jclass clss, jlong attr_id, jlong mem_type_id, jobjectArray buf) { herr_t status = -1; htri_t isStr = 0; htri_t isVlenStr = 0; htri_t isComplex = 0; if (buf == NULL) { h5nullArgument(env, "H5AreadVL: buf is NULL"); } /* end if */ else { isStr = H5Tdetect_class((hid_t)mem_type_id, H5T_STRING); if (H5Tget_class((hid_t)mem_type_id) == H5T_COMPOUND) { unsigned i; int nm = H5Tget_nmembers(mem_type_id); for(i = 0; i <nm; i++) { hid_t nested_tid = H5Tget_member_type((hid_t)mem_type_id, i); isComplex = H5Tdetect_class((hid_t)nested_tid, H5T_COMPOUND) || H5Tdetect_class((hid_t)nested_tid, H5T_VLEN); H5Tclose(nested_tid); } } else if (H5Tget_class((hid_t)mem_type_id) == H5T_VLEN) { isVlenStr = 1; /* strings created by H5Tvlen_create(H5T_C_S1) */ } if (isStr == 0 || isComplex>0 || isVlenStr) { status = H5AreadVL_asstr(env, (hid_t)attr_id, (hid_t)mem_type_id, buf); } else if (isStr > 0) { status = H5AreadVL_str(env, (hid_t)attr_id, (hid_t)mem_type_id, buf); } } /* end else */ return (jint)status; } /* end Java_hdf_hdf5lib_H5_H5Aread_1VL */
/*------------------------------------------------------------------------------ * Purpose: converts data value into strings * Parameters: H5Dataset* d-- The dataset its value to be vonverted. hid_t tid -- the datatype identifier * Return: Returns a non-negative value if successful; otherwise returns a negative value. *------------------------------------------------------------------------------ */ int H5Dataset_value_to_string(H5Dataset *d, hid_t tid, hid_t sid) { int ret_value=0; unsigned int i=0; char **strs; unsigned char *vp=NULL; h5str_t h5str; size_t offset=0, tsize=0, valuelen=0; void *value; assert(d); assert(d->value); value = d->value; vp = (unsigned char *)d->value; d->value = (char **)malloc(d->space.npoints*sizeof(char *)); assert(d->value); strs = (char**)d->value; offset = 0; tsize = H5Tget_size(tid); memset(&h5str, 0, sizeof(h5str_t)); h5str_new(&h5str, 4*tsize); d->nvalue = 0; for (i=0; i<d->space.npoints; i++) { h5str_empty(&h5str); ret_value = h5str_sprintf(&h5str, tid, vp + offset, " || "); if (ret_value > 0) { valuelen = strlen(h5str.s)+1; strs[i] = (char *)malloc(valuelen); strcpy(strs[i], h5str.s); /* d->nvalue += valuelen; XXXX changed by MW */ d->nvalue ++; } offset += tsize; } h5str_free(&h5str); /* reclaim memory allocated to store variable length data */ if (H5Tdetect_class(tid, H5T_VLEN) > 0) H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, value); free (value); return ret_value; }
/*------------------------------------------------------------------------- * Function: diff_datasetid * * Purpose: check for comparable datasets and read into a compatible * memory type * * Return: Number of differences found * * Programmer: Pedro Vicente, [email protected] * * Date: May 9, 2003 * * Modifications: * * * October 2006: Read by hyperslabs for big datasets. * * A threshold of H5TOOLS_MALLOCSIZE (128 MB) is the limit upon which I/O hyperslab is done * i.e., if the memory needed to read a dataset is greater than this limit, * then hyperslab I/O is done instead of one operation I/O * For each dataset, the memory needed is calculated according to * * memory needed = number of elements * size of each element * * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations * are done * * H5Dread( input_dataset1 ) * H5Dread( input_dataset2 ) * * with all elements in the datasets selected. If the memory needed is greater than * H5TOOLS_MALLOCSIZE, then the following operations are done instead: * * a strip mine is defined for each dimension k (a strip mine is defined as a * hyperslab whose size is memory manageable) according to the formula * * (1) strip_mine_size[k ] = MIN(dimension[k ], H5TOOLS_BUFSIZE / size of memory type) * * where H5TOOLS_BUFSIZE is a constant currently defined as 1MB. This formula assures * that for small datasets (small relative to the H5TOOLS_BUFSIZE constant), the strip * mine size k is simply defined as its dimension k, but for larger datasets the * hyperslab size is still memory manageable. * a cycle is done until the number of elements in the dataset is reached. In each * iteration, two parameters are defined for the function H5Sselect_hyperslab, * the start and size of each hyperslab, according to * * (2) hyperslab_size [k] = MIN(dimension[k] - hyperslab_offset[k], strip_mine_size [k]) * * where hyperslab_offset [k] is initially set to zero, and later incremented in * hyperslab_size[k] offsets. The reason for the operation * * dimension[k] - hyperslab_offset[k] * * in (2) is that, when using the strip mine size, it assures that the "remaining" part * of the dataset that does not fill an entire strip mine is processed. * *------------------------------------------------------------------------- */ hsize_t diff_datasetid( hid_t did1, hid_t did2, const char *obj1_name, const char *obj2_name, diff_opt_t *options) { hid_t sid1=-1; hid_t sid2=-1; hid_t f_tid1=-1; hid_t f_tid2=-1; hid_t m_tid1=-1; hid_t m_tid2=-1; size_t m_size1; size_t m_size2; H5T_sign_t sign1; H5T_sign_t sign2; int rank1; int rank2; hsize_t nelmts1; hsize_t nelmts2; hsize_t dims1[H5S_MAX_RANK]; hsize_t dims2[H5S_MAX_RANK]; hsize_t maxdim1[H5S_MAX_RANK]; hsize_t maxdim2[H5S_MAX_RANK]; const char *name1=NULL; /* relative names */ const char *name2=NULL; hsize_t storage_size1; hsize_t storage_size2; hsize_t nfound=0; /* number of differences found */ int can_compare=1; /* do diff or not */ void *buf1=NULL; void *buf2=NULL; void *sm_buf1=NULL; void *sm_buf2=NULL; size_t need; /* bytes needed for malloc */ int i; /* Get the dataspace handle */ if ( (sid1 = H5Dget_space(did1)) < 0 ) goto error; /* Get rank */ if ( (rank1 = H5Sget_simple_extent_ndims(sid1)) < 0 ) goto error; /* Get the dataspace handle */ if ( (sid2 = H5Dget_space(did2)) < 0 ) goto error; /* Get rank */ if ( (rank2 = H5Sget_simple_extent_ndims(sid2)) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid1,dims1,maxdim1) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid2,dims2,maxdim2) < 0 ) { goto error; } /*------------------------------------------------------------------------- * get the file data type *------------------------------------------------------------------------- */ /* Get the data type */ if ( (f_tid1 = H5Dget_type(did1)) < 0 ) goto error; /* Get the data type */ if ( (f_tid2 = H5Dget_type(did2)) < 0 ) { goto error; } /*------------------------------------------------------------------------- * check for empty datasets *------------------------------------------------------------------------- */ storage_size1=H5Dget_storage_size(did1); storage_size2=H5Dget_storage_size(did2); if (storage_size1==0 || storage_size2==0) { if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) parallel_print("Not comparable: <%s> or <%s> is an empty dataset\n", obj1_name, obj2_name); can_compare=0; options->not_cmp=1; } /*------------------------------------------------------------------------- * check for comparable TYPE and SPACE *------------------------------------------------------------------------- */ if (diff_can_type(f_tid1, f_tid2, rank1, rank2, dims1, dims2, maxdim1, maxdim2, obj1_name, obj2_name, options, 0)!=1) { can_compare=0; } /*------------------------------------------------------------------------- * memory type and sizes *------------------------------------------------------------------------- */ if ((m_tid1=h5tools_get_native_type(f_tid1)) < 0) goto error; if ((m_tid2=h5tools_get_native_type(f_tid2)) < 0) goto error; m_size1 = H5Tget_size( m_tid1 ); m_size2 = H5Tget_size( m_tid2 ); /*------------------------------------------------------------------------- * check for different signed/unsigned types *------------------------------------------------------------------------- */ sign1=H5Tget_sign(m_tid1); sign2=H5Tget_sign(m_tid2); if ( sign1 != sign2 ) { if ((options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) { parallel_print("Not comparable: <%s> has sign %s ", obj1_name, get_sign(sign1)); parallel_print("and <%s> has sign %s\n", obj2_name, get_sign(sign2)); } can_compare=0; options->not_cmp=1; } /*------------------------------------------------------------------------- * only attempt to compare if possible *------------------------------------------------------------------------- */ if (can_compare ) /* it is possible to compare */ { /*------------------------------------------------------------------------- * get number of elements *------------------------------------------------------------------------- */ nelmts1 = 1; for (i = 0; i < rank1; i++) { nelmts1 *= dims1[i]; } nelmts2 = 1; for (i = 0; i < rank2; i++) { nelmts2 *= dims2[i]; } assert(nelmts1==nelmts2); /*------------------------------------------------------------------------- * "upgrade" the smaller memory size *------------------------------------------------------------------------- */ if ( m_size1 != m_size2 ) { if ( m_size1 < m_size2 ) { H5Tclose(m_tid1); if ((m_tid1=h5tools_get_native_type(f_tid2)) < 0) goto error; m_size1 = H5Tget_size( m_tid1 ); } else { H5Tclose(m_tid2); if ((m_tid2=h5tools_get_native_type(f_tid1)) < 0) goto error; m_size2 = H5Tget_size( m_tid2 ); } } assert(m_size1==m_size2); /* print names */ if (obj1_name) { name1=diff_basename(obj1_name); } if (obj2_name) { name2=diff_basename(obj2_name); } /*------------------------------------------------------------------------- * read/compare *------------------------------------------------------------------------- */ need = (size_t)(nelmts1*m_size1); /* bytes needed */ if ( need < H5TOOLS_MALLOCSIZE) { buf1 = HDmalloc(need); buf2 = HDmalloc(need); } if ( buf1!=NULL && buf2!=NULL) { if ( H5Dread(did1,m_tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf1) < 0 ) goto error; if ( H5Dread(did2,m_tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf2) < 0 ) goto error; /* array diff */ nfound = diff_array(buf1, buf2, nelmts1, (hsize_t)0, rank1, dims1, options, name1, name2, m_tid1, did1, did2); } else /* possibly not enough memory, read/compare by hyperslabs */ { size_t p_type_nbytes = m_size1; /*size of memory type */ hsize_t p_nelmts = nelmts1; /*total selected elmts */ hsize_t elmtno; /*counter */ int carry; /*counter carry value */ unsigned int vl_data = 0; /*contains VL datatypes */ /* stripmine info */ hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ hsize_t sm_nbytes; /*bytes per stripmine */ hsize_t sm_nelmts; /*elements per stripmine*/ hid_t sm_space; /*stripmine data space */ /* hyperslab info */ hsize_t hs_offset[H5S_MAX_RANK]; /*starting offset */ hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ hsize_t hs_nelmts; /*elements in request */ hsize_t zero[8]; /*vector of zeros */ /* check if we have VL data in the dataset's datatype */ if (H5Tdetect_class(m_tid1, H5T_VLEN) == TRUE) vl_data = TRUE; /* * determine the strip mine size and allocate a buffer. The strip mine is * a hyperslab whose size is manageable. */ sm_nbytes = p_type_nbytes; for (i = rank1; i > 0; --i) { hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */ size = 1; sm_size[i - 1] = MIN(dims1[i - 1], size); sm_nbytes *= sm_size[i - 1]; assert(sm_nbytes > 0); } sm_buf1 = malloc((size_t)sm_nbytes); sm_buf2 = malloc((size_t)sm_nbytes); sm_nelmts = sm_nbytes / p_type_nbytes; sm_space = H5Screate_simple(1, &sm_nelmts, NULL); /* the stripmine loop */ memset(hs_offset, 0, sizeof hs_offset); memset(zero, 0, sizeof zero); for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) { /* calculate the hyperslab size */ if (rank1 > 0) { for (i = 0, hs_nelmts = 1; i < rank1; i++) { hs_size[i] = MIN(dims1[i] - hs_offset[i], sm_size[i]); hs_nelmts *= hs_size[i]; } if (H5Sselect_hyperslab(sid1, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if (H5Sselect_hyperslab(sid2, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL) < 0) goto error; } else { H5Sselect_all(sid1); H5Sselect_all(sid2); H5Sselect_all(sm_space); hs_nelmts = 1; } /* rank */ if ( H5Dread(did1,m_tid1,sm_space,sid1,H5P_DEFAULT,sm_buf1) < 0 ) goto error; if ( H5Dread(did2,m_tid2,sm_space,sid2,H5P_DEFAULT,sm_buf2) < 0 ) goto error; /* get array differences. in the case of hyperslab read, increment the number of differences found in each hyperslab and pass the position at the beggining for printing */ nfound += diff_array(sm_buf1, sm_buf2, hs_nelmts, elmtno, rank1, dims1, options, name1, name2, m_tid1, did1, did2); /* reclaim any VL memory, if necessary */ if(vl_data) { H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1); H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2); } /* calculate the next hyperslab offset */ for (i = rank1, carry = 1; i > 0 && carry; --i) { hs_offset[i - 1] += hs_size[i - 1]; if (hs_offset[i - 1] == dims1[i - 1]) hs_offset[i - 1] = 0; else carry = 0; } /* i */ } /* elmtno */ H5Sclose(sm_space); /* free */ if (sm_buf1!=NULL) { free(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { free(sm_buf2); sm_buf2=NULL; } } /* hyperslab read */ }/*can_compare*/ /*------------------------------------------------------------------------- * compare attributes * the if condition refers to cases when the dataset is a referenced object *------------------------------------------------------------------------- */ if (obj1_name) { nfound += diff_attr(did1,did2,obj1_name,obj2_name,options); } /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ /* free */ if (buf1!=NULL) { free(buf1); buf1=NULL; } if (buf2!=NULL) { free(buf2); buf2=NULL; } if (sm_buf1!=NULL) { free(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { free(sm_buf2); sm_buf2=NULL; } H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); } H5E_END_TRY; return nfound; error: options->err_stat=1; /* free */ if (buf1!=NULL) { free(buf1); buf1=NULL; } if (buf2!=NULL) { free(buf2); buf2=NULL; } if (sm_buf1!=NULL) { free(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { free(sm_buf2); sm_buf2=NULL; } /* disable error reporting */ H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); /* enable error reporting */ } H5E_END_TRY; return nfound; }
/**************************************************************** ** ** test_vlstring_type(): Test VL string type. ** Tests if VL string is treated as string. ** ****************************************************************/ static void test_vlstring_type(void) { hid_t fid; /* HDF5 File IDs */ hid_t tid_vlstr; H5T_cset_t cset; H5T_str_t pad; htri_t vl_str; /* Whether string is VL */ herr_t ret; /* Output message about test being performed */ MESSAGE(5, ("Testing VL String type\n")); /* Open file */ fid = H5Fopen(DATAFILE, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Create a datatype to refer to */ tid_vlstr = H5Tcopy(H5T_C_S1); CHECK(tid_vlstr, FAIL, "H5Tcopy"); /* Change padding and verify it */ ret = H5Tset_strpad(tid_vlstr, H5T_STR_NULLPAD); CHECK(ret, FAIL, "H5Tset_strpad"); pad = H5Tget_strpad(tid_vlstr); VERIFY(pad, H5T_STR_NULLPAD, "H5Tget_strpad"); /* Convert to variable-length string */ ret = H5Tset_size(tid_vlstr, H5T_VARIABLE); CHECK(ret, FAIL, "H5Tset_size"); /* Check if datatype is VL string */ ret = H5Tget_class(tid_vlstr); VERIFY(ret, H5T_STRING, "H5Tget_class"); ret = H5Tis_variable_str(tid_vlstr); VERIFY(ret, TRUE, "H5Tis_variable_str"); /* Verify that the class detects as a string */ vl_str = H5Tdetect_class(tid_vlstr, H5T_STRING); CHECK(vl_str, FAIL, "H5Tdetect_class"); VERIFY(vl_str, TRUE, "H5Tdetect_class"); /* Check default character set and padding */ cset = H5Tget_cset(tid_vlstr); VERIFY(cset, H5T_CSET_ASCII, "H5Tget_cset"); pad = H5Tget_strpad(tid_vlstr); VERIFY(pad, H5T_STR_NULLPAD, "H5Tget_strpad"); /* Commit variable-length string datatype to storage */ ret = H5Tcommit2(fid, VLSTR_TYPE, tid_vlstr, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(ret, FAIL, "H5Tcommit2"); /* Close datatype */ ret = H5Tclose(tid_vlstr); CHECK(ret, FAIL, "H5Tclose"); tid_vlstr = H5Topen2(fid, VLSTR_TYPE, H5P_DEFAULT); CHECK(tid_vlstr, FAIL, "H5Topen2"); ret = H5Tclose(tid_vlstr); CHECK(ret, FAIL, "H5Tclose"); ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); fid = H5Fopen(DATAFILE, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Open the variable-length string datatype just created */ tid_vlstr = H5Topen2(fid, VLSTR_TYPE, H5P_DEFAULT); CHECK(tid_vlstr, FAIL, "H5Topen2"); /* Verify character set and padding */ cset = H5Tget_cset(tid_vlstr); VERIFY(cset, H5T_CSET_ASCII, "H5Tget_cset"); pad = H5Tget_strpad(tid_vlstr); VERIFY(pad, H5T_STR_NULLPAD, "H5Tget_strpad"); /* Close datatype and file */ ret = H5Tclose(tid_vlstr); CHECK(ret, FAIL, "H5Tclose"); ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); } /* end test_vlstring_type() */