static int read_file(char *filename) { #define CWP "cwp" #define XLEN 3712 #define YLEN 3712 int ncid, varid; struct timeval start_time, end_time, diff_time; short *data; int time_us; printf("**** reading file %s\n", filename); if (gettimeofday(&start_time, NULL)) ERR_RET; if(complain(nc_open(filename, NC_NOWRITE, &ncid))) ERR_RET; if (gettimeofday(&end_time, NULL)) ERR_RET; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR_RET; time_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec; printf("File open time (us): %d\n", (int)time_us); if (!(data = malloc(sizeof(short) * XLEN * YLEN))) ERR; if (gettimeofday(&start_time, NULL)) ERR_RET; if (nc_inq_varid(ncid, CWP, &varid)) ERR; if (nc_get_var_short(ncid, varid, data)) ERR; if (gettimeofday(&end_time, NULL)) ERR_RET; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR_RET; time_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec; printf("Data read time (us): %d\n", (int)time_us); free(data); if (nc_close(ncid)) ERR_RET; return 0; }
int main() { printf("\n*** Checking many attributes in HDF5 file.\n"); printf("*** Checking some more simple atts...\n"); { #define NUM_ATTS 10000 hid_t fcpl_id, hdfid, grpid; hid_t spaceid, attid1; int one = 1; hsize_t dims[1] = {1}; int i; char name[NC_MAX_NAME]; struct timeval start_time, end_time, diff_time; double sec; /* Create a HDF5 file. */ if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0) ERR; if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED)) < 0) ERR; if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED)) < 0) ERR; if ((hdfid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT)) < 0) ERR; if (H5Pclose(fcpl_id) < 0) ERR; /* Open the root group. */ if ((grpid = H5Gopen2(hdfid, "/", H5P_DEFAULT)) < 0) ERR; if (gettimeofday(&start_time, NULL)) ERR; /* Write an attribute. */ if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0) ERR; for (i = 0; i < NUM_ATTS; i++) { sprintf(name, "att_%d", i); if ((attid1 = H5Acreate2(grpid, name, H5T_NATIVE_INT, spaceid, H5P_DEFAULT, H5P_DEFAULT)) < 0) ERR; if (H5Awrite(attid1, H5T_NATIVE_INT, &one) < 0) ERR; /* if (H5Aclose(attid1) < 0) ERR;*/ if((i + 1) % 1000 == 0) { /* only print every 1000th attribute name */ if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec; printf("%i\t%.3g sec\n", i + 1, sec); } } /* Close everything. */ if (H5Sclose(spaceid) < 0) ERR; if (H5Gclose(grpid) < 0) ERR; if (H5Fclose(hdfid) < 0) ERR; } SUMMARIZE_ERR; FINAL_RESULTS; }
int main(int argc, char **argv) { struct timeval start_time, end_time, diff_time; double sec; int nitem = 1000; /* default number of objects of each type */ int i; int ncid; int data[] = {42}; int g, grp, numgrp; char gname[16]; int a, numatt, an, aleft, natts; if(argc > 2) { /* Usage */ printf("NetCDF performance test, writing many groups, variables, and attributes.\n"); printf("Usage:\t%s [N]\n", argv[0]); printf("\tN: number of objects\n"); return(0); } for(i = 1; i < argc; i++) { nitem = atoi(argv[i]); } /* create new file */ if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; /* create N group/global attributes, printing time after every 100. * Because only NC_MAX_ATTRS are permitted per group, create the * necessary number of groups to hold nitem attributes. */ numatt = nitem; a = 1; numgrp = (numatt - 1) / NC_MAX_ATTRS + 1; aleft = numatt - (NC_MAX_ATTRS * (numgrp - 1)); if (gettimeofday(&start_time, NULL)) ERR; for(g = 1; g < numgrp + 1; g++) { sprintf(gname, "group%d", g); if (nc_def_grp(ncid, gname, &grp)) ERR; natts = g < numgrp ? NC_MAX_ATTRS : aleft; /* leftovers on last time through */ for(an = 1; an < natts + 1; an++) { char aname[20]; sprintf(aname, "attribute%d", a); if (nc_put_att_int(grp, NC_GLOBAL, aname, NC_INT, 1, data)) ERR; if(a%100 == 0) { /* only print every 100th attribute name */ if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec; printf("%s/%s\t%.3g sec\n", gname, aname, sec); } a++; } } nc_close(ncid); return(0); }
static int read_file(char *filename) { int ncid; struct timeval start_time, end_time, diff_time; int time_us; printf("**** reading file %s\n", filename); if (gettimeofday(&start_time, NULL)) ERR_RET; if (nc_open(filename, NC_NOWRITE, &ncid)) ERR_RET; if (gettimeofday(&end_time, NULL)) ERR_RET; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR_RET; time_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec; printf("File open time (us): %d\n", (int)time_us); if (nc_close(ncid)) ERR_RET; return 0; }
int main(int argc, char **argv) { extern int optind; extern int opterr; extern char *optarg; int c, header = 0, verbose = 0, timeseries = 0; int ncid, varid, storage; char name_in[NC_MAX_NAME + 1]; size_t len; size_t cs[NDIMS3] = {0, 0, 0}; int cache = MEGABYTE; int ndims, dimid[NDIMS3]; float hor_data[LAT_LEN * LON_LEN]; int read_1_us, avg_read_us; float ts_data[TIME_LEN]; size_t start[NDIMS3], count[NDIMS3]; int deflate, shuffle, deflate_level; struct timeval start_time, end_time, diff_time; while ((c = getopt(argc, argv, "vhtc:")) != EOF) switch(c) { case 'v': verbose++; break; case 'h': header++; break; case 't': timeseries++; break; case 'c': sscanf(optarg, "%d", &cache); break; case '?': usage(); return 1; } argc -= optind; argv += optind; /* If no file arguments left, report and exit */ if (argc < 1) { printf("no file specified\n"); return 0; } /* Print the header if desired. */ if (header) { printf("cs[0]\tcs[1]\tcs[2]\tcache(MB)\tdeflate\tshuffle"); if (timeseries) printf("\t1st_read_ser(us)\tavg_read_ser(us)\n"); else printf("\t1st_read_hor(us)\tavg_read_hor(us)\n"); } #define PREEMPTION .75 /* Also tried NELEMS of 2500009*/ #define NELEMS 7919 if (nc_set_chunk_cache(cache, NELEMS, PREEMPTION)) ERR; if (nc_open(argv[0], 0, &ncid)) ERR; /* Check to make sure that all the dimension information is * correct. */ if (nc_inq_varid(ncid, DATA_VAR_NAME, &varid)) ERR; if (nc_inq_dim(ncid, LON_DIMID, name_in, &len)) ERR; if (strcmp(name_in, "lon") || len != LON_LEN) ERR; if (nc_inq_dim(ncid, LAT_DIMID, name_in, &len)) ERR; if (strcmp(name_in, "lat") || len != LAT_LEN) ERR; if (nc_inq_dim(ncid, BNDS_DIMID, name_in, &len)) ERR; if (strcmp(name_in, "bnds") || len != BNDS_LEN) ERR; if (nc_inq_dim(ncid, TIME_DIMID, name_in, &len)) ERR; if (strcmp(name_in, "time") || len != TIME_LEN) ERR; if (nc_inq_var(ncid, varid, NULL, NULL, &ndims, dimid, NULL)) ERR; if (ndims != NDIMS3 || dimid[0] != TIME_DIMID || dimid[1] != LAT_DIMID || dimid[2] != LON_DIMID) ERR; /* Get info about the main data var. */ if (nc_inq_var_chunking(ncid, varid, &storage, cs)) ERR; if (nc_inq_var_deflate(ncid, varid, &shuffle, &deflate, &deflate_level)) ERR; if (timeseries) { /* Read the var as a time series. */ start[0] = 0; start[1] = 0; start[2] = 0; count[0] = TIME_LEN; count[1] = 1; count[2] = 1; /* Read the first timeseries. */ if (gettimeofday(&start_time, NULL)) ERR; if (nc_get_vara_float(ncid, varid, start, count, ts_data)) ERR_RET; if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; read_1_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec; /* Read all the rest. */ if (gettimeofday(&start_time, NULL)) ERR; for (start[1] = 0; start[1] < LAT_LEN; start[1]++) for (start[2] = 1; start[2] < LON_LEN; start[2]++) if (nc_get_vara_float(ncid, varid, start, count, ts_data)) ERR_RET; if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; avg_read_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec + read_1_us) / (LAT_LEN * LON_LEN); } else { /* Read the data variable in horizontal slices. */ start[0] = 0; start[1] = 0; start[2] = 0; count[0] = 1; count[1] = LAT_LEN; count[2] = LON_LEN; /* Read (and time) the first one. */ if (gettimeofday(&start_time, NULL)) ERR; if (nc_get_vara_float(ncid, varid, start, count, hor_data)) ERR_RET; if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; read_1_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec; /* Read (and time) all the rest. */ if (gettimeofday(&start_time, NULL)) ERR; for (start[0] = 1; start[0] < TIME_LEN; start[0]++) if (nc_get_vara_float(ncid, varid, start, count, hor_data)) ERR_RET; if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; avg_read_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec + read_1_us) / TIME_LEN; } /* Close file. */ if (nc_close(ncid)) ERR; /* Print results. */ printf("%d\t%d\t%d\t%.1f\t\t%d\t%d\t\t", (int)cs[0], (int)cs[1], (int)cs[2], (storage == NC_CHUNKED) ? (cache/(float)MEGABYTE) : 0, deflate, shuffle); if (timeseries) printf("%d\t\t%d\n", (int)read_1_us, (int)avg_read_us); else printf("%d\t\t%d\n", (int)read_1_us, (int)avg_read_us); return 0; }
int main(int argc, char **argv) { struct timeval start_time, end_time, diff_time; double sec; int nitem = 10000; /* default number of objects of each type */ int i; int ncid; int data[] = {42}; int g, grp, numgrp; char gname[16]; int v, var, numvar, vn, vleft, nvars; if(argc > 2) { /* Usage */ printf("NetCDF performance test, writing many groups and variables.\n"); printf("Usage:\t%s [N]\n", argv[0]); printf("\tN: number of objects\n"); return(0); } for(i = 1; i < argc; i++) { nitem = atoi(argv[i]); } /* create new file */ if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; if (gettimeofday(&start_time, NULL)) ERR; /* create N groups, printing time after every 1000 */ numgrp = nitem; for(g = 1; g < numgrp + 1; g++) { sprintf(gname, "group%d", g); if (nc_def_grp(ncid, gname, &grp)) ERR; if (nc_def_var(grp, "var", NC_INT, 0, NULL, &var)) ERR; if(nc_enddef (grp)) ERR; if(nc_put_var(grp, var, data)) ERR; if(g%1000 == 0) { /* only print every 1000th group name */ if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec; printf("%s\t%.3g sec\n", gname, sec); } } nc_close(ncid); /* create new file */ if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; /* create N variables, printing time after every 1000. Because only * NC_MAX_VARS are permitted per group, create the necessary number * of groups to hold nitem variables. */ numvar = nitem; v = 1; numgrp = (numvar - 1) / NC_MAX_VARS + 1; vleft = numvar - (NC_MAX_VARS * (numgrp - 1)); if (gettimeofday(&start_time, NULL)) ERR; for(g = 1; g < numgrp + 1; g++) { sprintf(gname, "group%d", g); if (nc_def_grp(ncid, gname, &grp)) ERR; nvars = g < numgrp ? NC_MAX_VARS : vleft; /* leftovers on last time through */ for(vn = 1; vn < nvars + 1; vn++) { int var; char vname[20]; sprintf(vname, "variable%d", v); if(nc_def_var(grp, vname, NC_INT, 0, NULL, &var)) ERR; if(nc_put_var(grp, var, data)) ERR; if(v%1000 == 0) { /* only print every 1000th variable name */ if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec; printf("%s/%s\t%.3g sec\n", gname, vname, sec); } v++; } } nc_close(ncid); return(0); }
int main(int argc, char **argv) { printf("\n*** Testing netcdf-4 file functions, some more.\n"); last_sbrk = sbrk(0); printf("*** testing lots of open files...\n"); { #define NUM_TRIES 6 int *ncid_in; int mem_used, mem_used2; /* int mem_per_file; */ int num_files[NUM_TRIES] = {1, 5, 10, 20, 35, 50}; char file_name[NUM_TRIES][NC_MAX_NAME + 1]; int num_vars[NUM_TRIES]; size_t cache_size[NUM_TRIES]; int mode[NUM_TRIES]; char mode_name[NUM_TRIES][8]; int ndims[NUM_TRIES]; int dim_len[NUM_TRIES][MAX_DIMS]; int dim_4d[MAX_DIMS] = {NC_UNLIMITED, 10, 100, 100}; char dimstr[30]; char chunkstr[30]; int num_recs[NUM_TRIES] = {1, 1, 1, 1, 1, 1}; struct timeval start_time, end_time, diff_time; struct timeval close_start_time, close_end_time, close_diff_time; int open_us, close_us, create_us; size_t chunksize[MAX_DIMS]; int storage; int d, f, t; printf("dims\t\tchunks\t\tformat\tnum_files\tcache(kb)\tnum_vars\tmem(kb)\t" "open_time(us/file)\tclose_time(us/file)\tcreate_time(us/file)\n"); for (t = 0; t < NUM_TRIES; t++) { strcpy(mode_name[t], "netcdf4"); mode[t] = NC_NETCDF4; cache_size[t] = 16000000; num_vars[t] = 10; ndims[t] = 4; for (d = 0; d < ndims[t]; d++) dim_len[t][d] = dim_4d[d]; /* Create sample files. */ if (gettimeofday(&start_time, NULL)) ERR; for (f = 0; f < num_files[t]; f++) { /* Set up filename. */ sprintf(file_name[t], "tst_files2_%d_%d.nc", t, f); if (create_sample_file(file_name[t], ndims[t], dim_len[t], num_vars[t], mode[t], num_recs[t])) ERR; /* How long did it take? */ if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; create_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec) / num_files[t]; } /* /\* Change the cache settings. *\/ */ /* if (nc_set_chunk_cache(cache_size[t], 20000, .75)) ERR; */ /* We need storage for an array of ncids. */ if (!(ncid_in = malloc(num_files[t] * sizeof(int)))) ERR; /* How much memory is in use now? */ if (get_mem_used1(&mem_used)) ERR; /* Open the first file to get chunksizes. */ if (gettimeofday(&start_time, NULL)) ERR; if (nc_open(file_name[t], 0, &ncid_in[0])) ERR; if (nc_inq_var_chunking(ncid_in[0], 0, &storage, chunksize)) ERR; /* Now reopen this file a large number of times. */ for (f = 1; f < num_files[t]; f++) if (nc_open(file_name[t], 0, &ncid_in[f])) ERR_RET; /* How long did it take per file? */ if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; open_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec) / num_files[t]; /* How much memory is in use by this process now? */ if (get_mem_used1(&mem_used2)) ERR; /* Close all netcdf files. */ if (gettimeofday(&close_start_time, NULL)) ERR; for (f = 0; f < num_files[t]; f++) if (nc_close(ncid_in[f])) ERR_RET; /* How long did it take to close all files? */ if (gettimeofday(&close_end_time, NULL)) ERR; if (nc4_timeval_subtract(&close_diff_time, &close_end_time, &close_start_time)) ERR; close_us = ((int)close_diff_time.tv_sec * MILLION + (int)close_diff_time.tv_usec) / num_files[t]; /* We're done with this. */ free(ncid_in); /* How much memory was used for each open file? */ /* mem_per_file = mem_used2/num_files[t]; */ /* Prepare the dimensions string. */ if (ndims[t] == MAX_DIMS) sprintf(dimstr, "%dx%dx%dx%d", dim_len[t][0], dim_len[t][1], dim_len[t][2], dim_len[t][3]); else sprintf(dimstr, "%dx%dx%d", dim_len[t][0], dim_len[t][1], dim_len[t][2]); /* Prepare the chunksize string. */ if (storage == NC_CHUNKED) { if (ndims[t] == MAX_DIMS) sprintf(chunkstr, "%dx%dx%dx%d", (int)chunksize[0], (int)chunksize[1], (int)chunksize[2], (int)chunksize[3]); else sprintf(chunkstr, "%dx%dx%d", (int)chunksize[0], (int)chunksize[1], (int)chunksize[2]); } else strcpy(chunkstr, "contig "); /* Output results. */ printf("%s\t%s\t%s\t%d\t\t%d\t\t%d\t\t%d\t\t%d\t\t%d\t\t%d\n", dimstr, chunkstr, mode_name[t], num_files[t], (int)(cache_size[t]/1024), num_vars[t], mem_used2, open_us, close_us, create_us); } } SUMMARIZE_ERR; printf("Test for memory consumption...\n"); { #define NUM_TRIES_100 100 int ncid, i; int mem_used, mem_used1, mem_used2; get_mem_used2(&mem_used); mem_used1 = mem_used; mem_used2 = mem_used; printf("start: memuse= %d\t%d\t%d \n",mem_used, mem_used1, mem_used2); printf("bef_open\taft_open\taft_close\tused_open\tused_closed\n"); for (i=0; i < NUM_TRIES_100; i++) { /* Open the file. NC_NOWRITE tells netCDF we want read-only access * to the file.*/ get_mem_used2(&mem_used); nc_set_chunk_cache(10,10,.5); if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; get_mem_used2(&mem_used1); /* Close the file, freeing all resources. ???? */ if (nc_close(ncid)) ERR; get_mem_used2(&mem_used2); if (mem_used2 - mem_used) printf("try %d - %d\t\t%d\t\t%d\t\t%d\t\t%d \n", i, mem_used, mem_used1, mem_used2, mem_used1 - mem_used, mem_used2 - mem_used); } } SUMMARIZE_ERR; FINAL_RESULTS; }