/***************************************************************************** This function generates attributes, groups, and datasets of many types. Parameters: fname: file_name. ngrps: number of top level groups. ndsets: number of datasets. attrs: number of attributes. nrow: number of rows in a dataset. chunk: chunk size (single number). vlen: max vlen size. comp: use latest format. latest: use gzip comnpression. Return: Non-negative on success/Negative on failure Programmer: Peter Cao <*****@*****.**>, Jan. 2013 ****************************************************************************/ herr_t create_perf_test_file(const char *fname, int ngrps, int ndsets, int nattrs, hsize_t nrows, hsize_t dim0, hsize_t chunk, int vlen, int compressed, int latest) { int i, j, k; hid_t fid, sid_null, sid_scalar, sid_1d, sid_2d, did, aid, sid_2, sid_large, fapl=H5P_DEFAULT, dcpl=H5P_DEFAULT, gid1, gid2, cmp_tid, tid_str, tid_enum, tid_array_f, tid_vlen_i, tid_vlen_s; char name[32], tmp_name1[32], tmp_name2[32], tmp_name3[32]; hsize_t dims[1]={dim0}, dims2d[2]={dim0, (dim0/4+1)}, dims_array[1]={FIXED_LEN}, dim1[1]={2}; char *enum_names[4] = {"SOLID", "LIQUID", "GAS", "PLASMA"}; test_comp_t *buf_comp=NULL, *buf_comp_large=NULL; int *buf_int=NULL; float (*buf_float_a)[FIXED_LEN]=NULL; double **buf_double2d=NULL; hvl_t *buf_vlen_i=NULL; char (*buf_str)[FIXED_LEN]; char **buf_vlen_s=NULL; hobj_ref_t buf_ref[2]; hdset_reg_ref_t buf_reg_ref[2]; size_t offset, len; herr_t status; char *names[NTYPES] = { "int", "ulong", "float", "double", "fixed string", "enum", "fixed float array", "vlen int array", "vlen strings"}; hid_t types[NTYPES] = { H5T_NATIVE_INT, H5T_NATIVE_UINT64, H5T_NATIVE_FLOAT, H5T_NATIVE_DOUBLE, tid_str, tid_enum, tid_array_f, tid_vlen_i, tid_vlen_s}; hsize_t coords[4][2] = { {0, 1}, {3, 5}, {1, 0}, {2, 4}}, start=0, stride=1, count=1; if (nrows < NROWS) nrows = NROWS; if (ngrps<NGROUPS) ngrps=NGROUPS; if (ndsets<NDSETS) ndsets=NDSETS; if (nattrs<NATTRS) nattrs=NATTRS; if (dim0<DIM0) dim0=DIM0; if (chunk>dim0) chunk=dim0/4; if (chunk<1) chunk = 1; if (vlen<1) vlen = MAXVLEN; /* create fixed string datatype */ types[4] = tid_str = H5Tcopy (H5T_C_S1); H5Tset_size (tid_str, FIXED_LEN); /* create enum datatype */ types[5] = tid_enum = H5Tenum_create(H5T_NATIVE_INT); for (i = (int) SOLID; i <= (int) PLASMA; i++) { phase_t val = (phase_t) i; status = H5Tenum_insert (tid_enum, enum_names[i], &val); } /* create float array datatype */ types[6] = tid_array_f = H5Tarray_create (H5T_NATIVE_FLOAT, 1, dims_array); /* create variable length integer datatypes */ types[7] = tid_vlen_i = H5Tvlen_create (H5T_NATIVE_INT); /* create variable length string datatype */ types[8] = tid_vlen_s = H5Tcopy (H5T_C_S1); H5Tset_size (tid_vlen_s, H5T_VARIABLE); /* create compound datatypes */ cmp_tid = H5Tcreate (H5T_COMPOUND, sizeof (test_comp_t)); offset = 0; for (i=0; i<NTYPES-2; i++) { H5Tinsert(cmp_tid, names[i], offset, types[i]); offset += H5Tget_size(types[i]); } H5Tinsert(cmp_tid, names[7], offset, types[7]); offset += sizeof (hvl_t); H5Tinsert(cmp_tid, names[8], offset, types[8]); /* create dataspace */ sid_1d = H5Screate_simple (1, dims, NULL); sid_2d = H5Screate_simple (2, dims2d, NULL); sid_2 = H5Screate_simple (1, dim1, NULL); sid_large = H5Screate_simple (1, &nrows, NULL); sid_null = H5Screate (H5S_NULL); sid_scalar = H5Screate (H5S_SCALAR); /* create fid access property */ fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_libver_bounds (fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); /* create dataset creation property */ dcpl = H5Pcreate (H5P_DATASET_CREATE); /* set dataset chunk */ if (chunk>0) { H5Pset_chunk (dcpl, 1, &chunk); } /* set dataset compression */ if (compressed) { if (chunk<=0) { chunk = dim0/10+1;; H5Pset_chunk (dcpl, 1, &chunk); } H5Pset_shuffle (dcpl); H5Pset_deflate (dcpl, 6); } /* allocate buffers */ buf_comp = (test_comp_t *)calloc(dim0, sizeof(test_comp_t)); buf_comp_large = (test_comp_t *)calloc(nrows, sizeof(test_comp_t)); buf_int = (int *)calloc(dim0, sizeof(int)); buf_float_a = malloc(dim0*sizeof(*buf_float_a)); buf_vlen_i = (hvl_t *)calloc(dim0, sizeof (hvl_t)); buf_vlen_s = (char **)calloc(dim0, sizeof(char *)); buf_str = malloc(dim0*sizeof (*buf_str)); /* allocate array of doulbe pointers */ buf_double2d = (double **)calloc(dims2d[0],sizeof(double *)); /* allocate a contigous chunk of memory for the data */ buf_double2d[0] = (double *)calloc( dims2d[0]*dims2d[1],sizeof(double) ); /* assign memory city to pointer array */ for (i=1; i <dims2d[0]; i++) buf_double2d[i] = buf_double2d[0]+i*dims2d[1]; /* fill buffer values */ len = 1; for (i=0; i<dims[0]; i++) { buf_comp[i].i = buf_int[i] = i-2147483648; buf_comp[i].l = 0xffffffffffffffff-i; buf_comp[i].f = 1.0/(i+1.0); buf_comp[i].d = 987654321.0*i+1.0/(i+1.0); buf_comp[i].e = (phase_t) (i % (int) (PLASMA + 1)); for (j=0; j<FIXED_LEN; j++) { buf_comp[i].f_array[j] = buf_float_a[i][j] = i*100+j; buf_str[i][j] = 'a' + (i%26); } buf_str[i][FIXED_LEN-1] = 0; strcpy(buf_comp[i].s, buf_str[i]); len = (1-cos(i/8.0))/2*vlen+1; if (!i) len = vlen; buf_vlen_i[i].len = len; buf_vlen_i[i].p = (int *)calloc(len, sizeof(int)); for (j=0; j<len; j++) ((int*)(buf_vlen_i[i].p))[j] = i*100+j; buf_comp[i].i_vlen = buf_vlen_i[i]; buf_vlen_s[i] = (char *)calloc(len, sizeof(char)); for (j=0; j<len-1; j++) buf_vlen_s[i][j] = j%26+'A'; buf_comp[i].s_vlen = buf_vlen_s[i]; for (j=0; j<dims2d[1]; j++) buf_double2d[i][j] = i+j/10000.0; } for (i=0; i<nrows; i++) { buf_comp_large[i].i = i-2147483648; buf_comp_large[i].l = 0xffffffffffffffff-i; buf_comp_large[i].f = 1.0/(i+1.0); buf_comp_large[i].d = 987654321.0*i+1.0/(i+1.0); buf_comp_large[i].e = (phase_t) (i % (int) (PLASMA + 1)); for (j=0; j<FIXED_LEN-1; j++) { buf_comp_large[i].f_array[j] = i*100+j; buf_comp_large[i].s[j] = 'a' + (i%26); } len = i%vlen+1; buf_comp_large[i].i_vlen.len = len; buf_comp_large[i].i_vlen.p = (int *)calloc(len, sizeof(int)); for (j=0; j<len; j++) ((int*)(buf_comp_large[i].i_vlen.p))[j] = i*100+j; buf_comp_large[i].s_vlen = (char *)calloc(i+2, sizeof(char)); for (j=0; j<i+1; j++) (buf_comp_large[i].s_vlen)[j] = j%26+'A'; } /* create file */ if (latest) fid = H5Fcreate (fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); else fid = H5Fcreate (fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); add_attrs(fid, 0); sprintf(name, "a cmp ds of %d rows", nrows); did = H5Dcreate (fid, name, cmp_tid, sid_large, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite (did, cmp_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_comp_large); add_attrs(did, 0); H5Dclose(did); // /* add attributes*/ gid1 = H5Gcreate (fid, "attributes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (nattrs<1) nattrs = 1; i=0; while (i<nattrs) i += add_attrs(gid1, i); H5Gclose(gid1); /* add many sub groups to a group*/ gid1 = H5Gcreate (fid, "groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); add_attrs(gid1, 0); for (i=0; i<ngrps; i++) { /* create sub groups */ sprintf(name, "g%02d", i); gid2 = H5Gcreate (gid1, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (i<10) add_attrs(gid2, 0); H5Gclose(gid2); } H5Gclose(gid1); /* add many datasets to a group */ gid1 = H5Gcreate (fid, "datasets", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); add_attrs(gid1, 0); for (j=0; j<ndsets; j+=12) { /* 1 add a null dataset */ sprintf(name, "%05d null dataset", j); did = H5Dcreate (gid1, name, H5T_STD_I32LE, sid_null, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (!j) add_attrs(did, j); H5Dclose(did); /* 2 add scalar int point */ sprintf(name, "%05d scalar int point", j); did = H5Dcreate (gid1, name, H5T_NATIVE_INT, sid_scalar, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite (did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &j); if (!j) add_attrs(did, j); H5Dclose(did); /* 3 scalar vlen string */ sprintf(name, "%05d scalar vlen string", j); did = H5Dcreate (gid1, name, tid_vlen_s, sid_scalar, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite (did, tid_vlen_s, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buf_vlen_s[0]); if (!j) add_attrs(did, j); H5Dclose(did); /* 4 add fixed-length float array */ sprintf(name, "%05d fixed-length float array", j); did = H5Dcreate (gid1, name, tid_array_f, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite (did, tid_array_f, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_float_a); if (!j) add_attrs(did, j); H5Dclose(did); /* 5 add fixed-length strings */ sprintf(name, "%05d fixed-length strings", j); did = H5Dcreate (gid1, name, tid_str, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite (did, tid_str, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_str); if (!j) add_attrs(did, j); H5Dclose(did); /* 6 add compound data */ sprintf(name, "%05d compund data", j); did = H5Dcreate (gid1, name, cmp_tid, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite (did, cmp_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_comp); if (!j) add_attrs(did, j); H5Dclose(did); /* 7 add 2D double */ sprintf(name, "%05d 2D double", j); strcpy (tmp_name1, name); did = H5Dcreate (gid1, name, H5T_NATIVE_DOUBLE, sid_2d, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite (did, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_double2d[0]); if (!j) add_attrs(did, j); H5Dclose(did); /* 8 add 1D int array */ sprintf(name, "%05d 1D int array", j); did = H5Dcreate (gid1, name, H5T_NATIVE_INT, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite (did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_int); if (!j) add_attrs(did, j); H5Dclose(did); /* 9 add vlen int array */ sprintf(name, "%05d vlen int array", j); strcpy (tmp_name2, name); did = H5Dcreate (gid1, name, tid_vlen_i, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite (did, tid_vlen_i, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_vlen_i); if (!j) add_attrs(did, j); H5Dclose(did); /* 10 add vlen strings */ sprintf(name, "%05d vlen strings", j); strcpy (tmp_name3, name); did = H5Dcreate (gid1, name, tid_vlen_s, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite (did, tid_vlen_s, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_vlen_s); if (!j) add_attrs(did, j); H5Dclose(did); /* 11 add object refs */ H5Rcreate(&buf_ref[0],gid1, ".", H5R_OBJECT, -1); H5Rcreate(&buf_ref[1],gid1, tmp_name3, H5R_OBJECT, -1); sprintf(name, "%05d obj refs", j); did = H5Dcreate (gid1, name, H5T_STD_REF_OBJ, sid_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite (did, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_ref); if (!j) add_attrs(did, j); H5Dclose(did); /* 12 add region refs */ H5Sselect_elements (sid_2d, H5S_SELECT_SET, 4, coords[0]); H5Rcreate(&buf_reg_ref[0],gid1, tmp_name1, H5R_DATASET_REGION, sid_2d); H5Sselect_none(sid_2d); count = dims[0]/2+1; H5Sselect_hyperslab (sid_1d, H5S_SELECT_SET, &start, &stride, &count,NULL); H5Rcreate(&buf_reg_ref[1],gid1, tmp_name2, H5R_DATASET_REGION, sid_1d); H5Sselect_none(sid_1d); sprintf(name, "%05d region refs", j); did = H5Dcreate (gid1, name, H5T_STD_REF_DSETREG, sid_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite (did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_reg_ref); if (!j) add_attrs(did, j); H5Dclose(did); } H5Gclose(gid1); H5Tclose (tid_array_f); H5Tclose (tid_vlen_i); H5Tclose (tid_vlen_s); H5Tclose (tid_enum); H5Tclose (tid_str); H5Tclose (cmp_tid); H5Pclose (dcpl); H5Pclose (fapl); H5Sclose (sid_1d); H5Sclose (sid_2d); H5Sclose (sid_2); H5Sclose (sid_large); H5Sclose (sid_null); H5Sclose (sid_scalar); H5Fclose (fid); for (i=0; i<dims[0]; i++) { if (buf_vlen_i[i].p) free(buf_vlen_i[i].p); if (buf_vlen_s[i]) free(buf_vlen_s[i]); } for (i=0; i<nrows; i++) { if (buf_comp_large[i].i_vlen.p) free(buf_comp_large[i].i_vlen.p); if (buf_comp_large[i].s_vlen) free(buf_comp_large[i].s_vlen); } free (buf_comp); free (buf_comp_large); free (buf_int); free (buf_float_a); free (buf_double2d[0]); free (buf_double2d); free (buf_str); free(buf_vlen_i); free(buf_vlen_s); return 0; }
int main() { printf("\n*** Creating file with datasets & attributes that have scalar dataspaces..."); { hid_t fileid; hid_t fcplid; hid_t dsetid; hid_t dcplid; hid_t scalar_spaceid; hid_t vlstr_typeid, fixstr_typeid; hid_t attid; /* Create scalar dataspace */ if ((scalar_spaceid = H5Screate(H5S_SCALAR)) < 0) ERR; /* Set creation ordering for file, so we can revise its contents later */ if ((fcplid = H5Pcreate(H5P_FILE_CREATE)) < 0) ERR; if (H5Pset_link_creation_order(fcplid, H5P_CRT_ORDER_TRACKED) < 0) ERR; if (H5Pset_attr_creation_order(fcplid, H5P_CRT_ORDER_TRACKED) < 0) ERR; /* Create new file, using default properties */ if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, fcplid, H5P_DEFAULT)) < 0) ERR; /* Close file creation property list */ if (H5Pclose(fcplid) < 0) ERR; /* Create variable-length string datatype */ if ((vlstr_typeid = H5Tcreate(H5T_STRING, (size_t)H5T_VARIABLE)) < 0) ERR; /* Create fixed-length string datatype */ if ((fixstr_typeid = H5Tcreate(H5T_STRING, (size_t)10)) < 0) ERR; /* Set creation ordering for dataset, so we can revise its contents later */ if ((dcplid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR; if (H5Pset_attr_creation_order(dcplid, H5P_CRT_ORDER_TRACKED) < 0) ERR; /* Create scalar dataset with VL string datatype */ if ((dsetid = H5Dcreate2(fileid, VSTR_VAR1_NAME, vlstr_typeid, scalar_spaceid, H5P_DEFAULT, dcplid, H5P_DEFAULT)) < 0) ERR; /* Add attributes to dataset */ if (add_attrs(dsetid) < 0) ERR; /* Close VL string dataset */ if (H5Dclose(dsetid) < 0) ERR; /* Create scalar dataset with fixed-length string datatype */ if ((dsetid = H5Dcreate2(fileid, FSTR_VAR_NAME, fixstr_typeid, scalar_spaceid, H5P_DEFAULT, dcplid, H5P_DEFAULT)) < 0) ERR; /* Add attributes to dataset */ if (add_attrs(dsetid) < 0) ERR; /* Close fixed-length string dataset */ if (H5Dclose(dsetid) < 0) ERR; /* Create scalar dataset with native integer datatype */ if ((dsetid = H5Dcreate2(fileid, INT_VAR_NAME, H5T_NATIVE_INT, scalar_spaceid, H5P_DEFAULT, dcplid, H5P_DEFAULT)) < 0) ERR; /* Add attributes to dataset */ if (add_attrs(dsetid) < 0) ERR; /* Close native integer dataset */ if (H5Dclose(dsetid) < 0) ERR; /* Add attributes to root group */ if (add_attrs(fileid) < 0) ERR; /* Close dataset creation property list */ if (H5Pclose(dcplid) < 0) ERR; /* Close string datatypes */ if (H5Tclose(vlstr_typeid) < 0) ERR; if (H5Tclose(fixstr_typeid) < 0) ERR; /* Close rest */ if (H5Sclose(scalar_spaceid) < 0) ERR; if (H5Fclose(fileid) < 0) ERR; } SUMMARIZE_ERR; printf("*** Checking accessing file through netCDF-4 API..."); { int ncid, varid; size_t len; nc_type type; int ndims; char *vlstr; int x; if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; /* Check the global attributes are OK */ if (check_attrs(ncid, NC_GLOBAL) < 0) ERR; /* Verify that the VL string dataset is present and OK */ if (nc_inq_varid(ncid, VSTR_VAR1_NAME, &varid)) ERR; if (varid != 0) ERR; if (nc_inq_vartype(ncid, varid, &type)) ERR; if (type != NC_STRING) ERR; if (nc_inq_varndims(ncid, varid, &ndims)) ERR; if (ndims != 0) ERR; vlstr = NULL; if (nc_get_var(ncid, varid, &vlstr)) ERR; if (NULL != vlstr) ERR; /* Check the variable's attributes are OK */ if (check_attrs(ncid, varid) < 0) ERR; /* Verify that the fixed-length string dataset is present and OK */ if (nc_inq_varid(ncid, FSTR_VAR_NAME, &varid)) ERR; if (varid != 1) ERR; if (nc_inq_vartype(ncid, varid, &type)) ERR; if (type != NC_STRING) ERR; if (nc_inq_varndims(ncid, varid, &ndims)) ERR; if (ndims != 0) ERR; vlstr = NULL; if (nc_get_var(ncid, varid, &vlstr)) ERR; if ('\0' != *vlstr) ERR; free(vlstr); /* Check the variable's attributes are OK */ if (check_attrs(ncid, varid) < 0) ERR; /* Verify that the integer dataset is present and OK */ if (nc_inq_varid(ncid, INT_VAR_NAME, &varid)) ERR; if (varid != 2) ERR; if (nc_inq_vartype(ncid, varid, &type)) ERR; if (type != NC_INT) ERR; if (nc_inq_varndims(ncid, varid, &ndims)) ERR; if (ndims != 0) ERR; x = -1; if (nc_get_var(ncid, varid, &x)) ERR; if (0 != x) ERR; /* Check the variable's attributes are OK */ if (check_attrs(ncid, varid) < 0) ERR; if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; printf("*** Checking revising file through netCDF-4 API..."); { int ncid, varid; char *vlstr; if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; /* Write to the VL string variable */ if (nc_inq_varid(ncid, VSTR_VAR1_NAME, &varid)) ERR; vlstr = NULL; if (nc_put_var(ncid, varid, &vlstr)) ERR; vlstr = malloc(10); *vlstr = '\0'; if (nc_put_var(ncid, varid, &vlstr)) ERR; strcpy(vlstr, "foo"); if (nc_put_var(ncid, varid, &vlstr)) ERR; free(vlstr); /* Write to a VL string attribute */ vlstr = NULL; if (nc_put_att(ncid, varid, VSTR_ATT1_NAME, NC_STRING, 1, &vlstr)) ERR; vlstr = malloc(10); *vlstr = '\0'; if (nc_put_att(ncid, varid, VSTR_ATT1_NAME, NC_STRING, 1, &vlstr)) ERR; strcpy(vlstr, "foo"); if (nc_put_att(ncid, varid, VSTR_ATT1_NAME, NC_STRING, 1, &vlstr)) ERR; free(vlstr); /* Define a new VL string variable */ if (nc_def_var(ncid, VSTR_VAR2_NAME , NC_STRING, 0, NULL, &varid)) ERR; /* Write to the variable's fill-value */ vlstr = NULL; if (nc_put_att(ncid, varid, _FillValue, NC_STRING, 1, &vlstr)) ERR; vlstr = malloc(10); *vlstr = '\0'; if (nc_put_att(ncid, varid, _FillValue, NC_STRING, 1, &vlstr)) ERR; strcpy(vlstr, "foo"); if (nc_put_att(ncid, varid, _FillValue, NC_STRING, 1, &vlstr)) ERR; free(vlstr); if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; FINAL_RESULTS; }