static int
read_file(char *filename)
{
#define CWP "cwp"
#define XLEN 3712
#define YLEN 3712
int ncid, varid;
struct timeval start_time, end_time, diff_time;
short *data;
int time_us;
printf("**** reading file %s\n", filename);
if (gettimeofday(&start_time, NULL)) ERR_RET;
if(complain(nc_open(filename, NC_NOWRITE, &ncid))) ERR_RET;
if (gettimeofday(&end_time, NULL)) ERR_RET;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR_RET;
time_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
printf("File open time (us): %d\n", (int)time_us);
if (!(data = malloc(sizeof(short) * XLEN * YLEN))) ERR;
if (gettimeofday(&start_time, NULL)) ERR_RET;
if (nc_inq_varid(ncid, CWP, &varid)) ERR;
if (nc_get_var_short(ncid, varid, data)) ERR;
if (gettimeofday(&end_time, NULL)) ERR_RET;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR_RET;
time_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
printf("Data read time (us): %d\n", (int)time_us);
free(data);
if (nc_close(ncid))
ERR_RET;
return 0;
}
int
main()
{
printf("\n*** Checking many attributes in HDF5 file.\n");
printf("*** Checking some more simple atts...\n");
{
#define NUM_ATTS 10000
hid_t fcpl_id, hdfid, grpid;
hid_t spaceid, attid1;
int one = 1;
hsize_t dims[1] = {1};
int i;
char name[NC_MAX_NAME];
struct timeval start_time, end_time, diff_time;
double sec;
/* Create a HDF5 file. */
if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0) ERR;
if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
H5P_CRT_ORDER_INDEXED)) < 0) ERR;
if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
H5P_CRT_ORDER_INDEXED)) < 0) ERR;
if ((hdfid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT)) < 0) ERR;
if (H5Pclose(fcpl_id) < 0) ERR;
/* Open the root group. */
if ((grpid = H5Gopen2(hdfid, "/", H5P_DEFAULT)) < 0) ERR;
if (gettimeofday(&start_time, NULL)) ERR;
/* Write an attribute. */
if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0) ERR;
for (i = 0; i < NUM_ATTS; i++)
{
sprintf(name, "att_%d", i);
if ((attid1 = H5Acreate2(grpid, name, H5T_NATIVE_INT, spaceid,
H5P_DEFAULT, H5P_DEFAULT)) < 0) ERR;
if (H5Awrite(attid1, H5T_NATIVE_INT, &one) < 0) ERR;
/* if (H5Aclose(attid1) < 0) ERR;*/
if((i + 1) % 1000 == 0)
{ /* only print every 1000th attribute name */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec;
printf("%i\t%.3g sec\n", i + 1, sec);
}
}
/* Close everything. */
if (H5Sclose(spaceid) < 0) ERR;
if (H5Gclose(grpid) < 0) ERR;
if (H5Fclose(hdfid) < 0) ERR;
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}
int main(int argc, char **argv)
{
struct timeval start_time, end_time, diff_time;
double sec;
int nitem = 1000; /* default number of objects of each type */
int i;
int ncid;
int data[] = {42};
int g, grp, numgrp;
char gname[16];
int a, numatt, an, aleft, natts;
if(argc > 2) { /* Usage */
printf("NetCDF performance test, writing many groups, variables, and attributes.\n");
printf("Usage:\t%s [N]\n", argv[0]);
printf("\tN: number of objects\n");
return(0);
}
for(i = 1; i < argc; i++) {
nitem = atoi(argv[i]);
}
/* create new file */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
/* create N group/global attributes, printing time after every 100.
* Because only NC_MAX_ATTRS are permitted per group, create the
* necessary number of groups to hold nitem attributes. */
numatt = nitem;
a = 1;
numgrp = (numatt - 1) / NC_MAX_ATTRS + 1;
aleft = numatt - (NC_MAX_ATTRS * (numgrp - 1));
if (gettimeofday(&start_time, NULL))
ERR;
for(g = 1; g < numgrp + 1; g++) {
sprintf(gname, "group%d", g);
if (nc_def_grp(ncid, gname, &grp)) ERR;
natts = g < numgrp ? NC_MAX_ATTRS : aleft; /* leftovers on last time through */
for(an = 1; an < natts + 1; an++) {
char aname[20];
sprintf(aname, "attribute%d", a);
if (nc_put_att_int(grp, NC_GLOBAL, aname, NC_INT, 1, data)) ERR;
if(a%100 == 0) { /* only print every 100th attribute name */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec;
printf("%s/%s\t%.3g sec\n", gname, aname, sec);
}
a++;
}
}
nc_close(ncid);
return(0);
}
static int
read_file(char *filename)
{
int ncid;
struct timeval start_time, end_time, diff_time;
int time_us;
printf("**** reading file %s\n", filename);
if (gettimeofday(&start_time, NULL)) ERR_RET;
if (nc_open(filename, NC_NOWRITE, &ncid)) ERR_RET;
if (gettimeofday(&end_time, NULL)) ERR_RET;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR_RET;
time_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
printf("File open time (us): %d\n", (int)time_us);
if (nc_close(ncid))
ERR_RET;
return 0;
}
int
main(int argc, char **argv)
{
extern int optind;
extern int opterr;
extern char *optarg;
int c, header = 0, verbose = 0, timeseries = 0;
int ncid, varid, storage;
char name_in[NC_MAX_NAME + 1];
size_t len;
size_t cs[NDIMS3] = {0, 0, 0};
int cache = MEGABYTE;
int ndims, dimid[NDIMS3];
float hor_data[LAT_LEN * LON_LEN];
int read_1_us, avg_read_us;
float ts_data[TIME_LEN];
size_t start[NDIMS3], count[NDIMS3];
int deflate, shuffle, deflate_level;
struct timeval start_time, end_time, diff_time;
while ((c = getopt(argc, argv, "vhtc:")) != EOF)
switch(c)
{
case 'v':
verbose++;
break;
case 'h':
header++;
break;
case 't':
timeseries++;
break;
case 'c':
sscanf(optarg, "%d", &cache);
break;
case '?':
usage();
return 1;
}
argc -= optind;
argv += optind;
/* If no file arguments left, report and exit */
if (argc < 1)
{
printf("no file specified\n");
return 0;
}
/* Print the header if desired. */
if (header)
{
printf("cs[0]\tcs[1]\tcs[2]\tcache(MB)\tdeflate\tshuffle");
if (timeseries)
printf("\t1st_read_ser(us)\tavg_read_ser(us)\n");
else
printf("\t1st_read_hor(us)\tavg_read_hor(us)\n");
}
#define PREEMPTION .75
/* Also tried NELEMS of 2500009*/
#define NELEMS 7919
if (nc_set_chunk_cache(cache, NELEMS, PREEMPTION)) ERR;
if (nc_open(argv[0], 0, &ncid)) ERR;
/* Check to make sure that all the dimension information is
* correct. */
if (nc_inq_varid(ncid, DATA_VAR_NAME, &varid)) ERR;
if (nc_inq_dim(ncid, LON_DIMID, name_in, &len)) ERR;
if (strcmp(name_in, "lon") || len != LON_LEN) ERR;
if (nc_inq_dim(ncid, LAT_DIMID, name_in, &len)) ERR;
if (strcmp(name_in, "lat") || len != LAT_LEN) ERR;
if (nc_inq_dim(ncid, BNDS_DIMID, name_in, &len)) ERR;
if (strcmp(name_in, "bnds") || len != BNDS_LEN) ERR;
if (nc_inq_dim(ncid, TIME_DIMID, name_in, &len)) ERR;
if (strcmp(name_in, "time") || len != TIME_LEN) ERR;
if (nc_inq_var(ncid, varid, NULL, NULL, &ndims, dimid, NULL)) ERR;
if (ndims != NDIMS3 || dimid[0] != TIME_DIMID ||
dimid[1] != LAT_DIMID || dimid[2] != LON_DIMID) ERR;
/* Get info about the main data var. */
if (nc_inq_var_chunking(ncid, varid, &storage, cs)) ERR;
if (nc_inq_var_deflate(ncid, varid, &shuffle, &deflate,
&deflate_level)) ERR;
if (timeseries)
{
/* Read the var as a time series. */
start[0] = 0;
start[1] = 0;
start[2] = 0;
count[0] = TIME_LEN;
count[1] = 1;
count[2] = 1;
/* Read the first timeseries. */
if (gettimeofday(&start_time, NULL)) ERR;
if (nc_get_vara_float(ncid, varid, start, count, ts_data)) ERR_RET;
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
read_1_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
/* Read all the rest. */
if (gettimeofday(&start_time, NULL)) ERR;
for (start[1] = 0; start[1] < LAT_LEN; start[1]++)
for (start[2] = 1; start[2] < LON_LEN; start[2]++)
if (nc_get_vara_float(ncid, varid, start, count, ts_data)) ERR_RET;
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
avg_read_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec + read_1_us) /
(LAT_LEN * LON_LEN);
}
else
{
/* Read the data variable in horizontal slices. */
start[0] = 0;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = LAT_LEN;
count[2] = LON_LEN;
/* Read (and time) the first one. */
if (gettimeofday(&start_time, NULL)) ERR;
if (nc_get_vara_float(ncid, varid, start, count, hor_data)) ERR_RET;
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
read_1_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
/* Read (and time) all the rest. */
if (gettimeofday(&start_time, NULL)) ERR;
for (start[0] = 1; start[0] < TIME_LEN; start[0]++)
if (nc_get_vara_float(ncid, varid, start, count, hor_data)) ERR_RET;
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
avg_read_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec +
read_1_us) / TIME_LEN;
}
/* Close file. */
if (nc_close(ncid)) ERR;
/* Print results. */
printf("%d\t%d\t%d\t%.1f\t\t%d\t%d\t\t",
(int)cs[0], (int)cs[1], (int)cs[2],
(storage == NC_CHUNKED) ? (cache/(float)MEGABYTE) : 0,
deflate, shuffle);
if (timeseries)
printf("%d\t\t%d\n", (int)read_1_us, (int)avg_read_us);
else
printf("%d\t\t%d\n", (int)read_1_us, (int)avg_read_us);
return 0;
}
int main(int argc, char **argv)
{
struct timeval start_time, end_time, diff_time;
double sec;
int nitem = 10000; /* default number of objects of each type */
int i;
int ncid;
int data[] = {42};
int g, grp, numgrp;
char gname[16];
int v, var, numvar, vn, vleft, nvars;
if(argc > 2) { /* Usage */
printf("NetCDF performance test, writing many groups and variables.\n");
printf("Usage:\t%s [N]\n", argv[0]);
printf("\tN: number of objects\n");
return(0);
}
for(i = 1; i < argc; i++) {
nitem = atoi(argv[i]);
}
/* create new file */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (gettimeofday(&start_time, NULL))
ERR;
/* create N groups, printing time after every 1000 */
numgrp = nitem;
for(g = 1; g < numgrp + 1; g++) {
sprintf(gname, "group%d", g);
if (nc_def_grp(ncid, gname, &grp)) ERR;
if (nc_def_var(grp, "var", NC_INT, 0, NULL, &var)) ERR;
if(nc_enddef (grp)) ERR;
if(nc_put_var(grp, var, data)) ERR;
if(g%1000 == 0) { /* only print every 1000th group name */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec;
printf("%s\t%.3g sec\n", gname, sec);
}
}
nc_close(ncid);
/* create new file */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
/* create N variables, printing time after every 1000. Because only
* NC_MAX_VARS are permitted per group, create the necessary number
* of groups to hold nitem variables. */
numvar = nitem;
v = 1;
numgrp = (numvar - 1) / NC_MAX_VARS + 1;
vleft = numvar - (NC_MAX_VARS * (numgrp - 1));
if (gettimeofday(&start_time, NULL))
ERR;
for(g = 1; g < numgrp + 1; g++) {
sprintf(gname, "group%d", g);
if (nc_def_grp(ncid, gname, &grp)) ERR;
nvars = g < numgrp ? NC_MAX_VARS : vleft; /* leftovers on last time through */
for(vn = 1; vn < nvars + 1; vn++) {
int var;
char vname[20];
sprintf(vname, "variable%d", v);
if(nc_def_var(grp, vname, NC_INT, 0, NULL, &var)) ERR;
if(nc_put_var(grp, var, data)) ERR;
if(v%1000 == 0) { /* only print every 1000th variable name */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
sec = diff_time.tv_sec + 1.0e-6 * diff_time.tv_usec;
printf("%s/%s\t%.3g sec\n", gname, vname, sec);
}
v++;
}
}
nc_close(ncid);
return(0);
}
int
main(int argc, char **argv)
{
printf("\n*** Testing netcdf-4 file functions, some more.\n");
last_sbrk = sbrk(0);
printf("*** testing lots of open files...\n");
{
#define NUM_TRIES 6
int *ncid_in;
int mem_used, mem_used2;
/* int mem_per_file; */
int num_files[NUM_TRIES] = {1, 5, 10, 20, 35, 50};
char file_name[NUM_TRIES][NC_MAX_NAME + 1];
int num_vars[NUM_TRIES];
size_t cache_size[NUM_TRIES];
int mode[NUM_TRIES];
char mode_name[NUM_TRIES][8];
int ndims[NUM_TRIES];
int dim_len[NUM_TRIES][MAX_DIMS];
int dim_4d[MAX_DIMS] = {NC_UNLIMITED, 10, 100, 100};
char dimstr[30];
char chunkstr[30];
int num_recs[NUM_TRIES] = {1, 1, 1, 1, 1, 1};
struct timeval start_time, end_time, diff_time;
struct timeval close_start_time, close_end_time, close_diff_time;
int open_us, close_us, create_us;
size_t chunksize[MAX_DIMS];
int storage;
int d, f, t;
printf("dims\t\tchunks\t\tformat\tnum_files\tcache(kb)\tnum_vars\tmem(kb)\t"
"open_time(us/file)\tclose_time(us/file)\tcreate_time(us/file)\n");
for (t = 0; t < NUM_TRIES; t++)
{
strcpy(mode_name[t], "netcdf4");
mode[t] = NC_NETCDF4;
cache_size[t] = 16000000;
num_vars[t] = 10;
ndims[t] = 4;
for (d = 0; d < ndims[t]; d++)
dim_len[t][d] = dim_4d[d];
/* Create sample files. */
if (gettimeofday(&start_time, NULL)) ERR;
for (f = 0; f < num_files[t]; f++)
{
/* Set up filename. */
sprintf(file_name[t], "tst_files2_%d_%d.nc", t, f);
if (create_sample_file(file_name[t], ndims[t], dim_len[t], num_vars[t],
mode[t], num_recs[t])) ERR;
/* How long did it take? */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
create_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec) / num_files[t];
}
/* /\* Change the cache settings. *\/ */
/* if (nc_set_chunk_cache(cache_size[t], 20000, .75)) ERR; */
/* We need storage for an array of ncids. */
if (!(ncid_in = malloc(num_files[t] * sizeof(int)))) ERR;
/* How much memory is in use now? */
if (get_mem_used1(&mem_used)) ERR;
/* Open the first file to get chunksizes. */
if (gettimeofday(&start_time, NULL)) ERR;
if (nc_open(file_name[t], 0, &ncid_in[0])) ERR;
if (nc_inq_var_chunking(ncid_in[0], 0, &storage, chunksize)) ERR;
/* Now reopen this file a large number of times. */
for (f = 1; f < num_files[t]; f++)
if (nc_open(file_name[t], 0, &ncid_in[f])) ERR_RET;
/* How long did it take per file? */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
open_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec) / num_files[t];
/* How much memory is in use by this process now? */
if (get_mem_used1(&mem_used2)) ERR;
/* Close all netcdf files. */
if (gettimeofday(&close_start_time, NULL)) ERR;
for (f = 0; f < num_files[t]; f++)
if (nc_close(ncid_in[f])) ERR_RET;
/* How long did it take to close all files? */
if (gettimeofday(&close_end_time, NULL)) ERR;
if (nc4_timeval_subtract(&close_diff_time, &close_end_time, &close_start_time)) ERR;
close_us = ((int)close_diff_time.tv_sec * MILLION +
(int)close_diff_time.tv_usec) / num_files[t];
/* We're done with this. */
free(ncid_in);
/* How much memory was used for each open file? */
/* mem_per_file = mem_used2/num_files[t]; */
/* Prepare the dimensions string. */
if (ndims[t] == MAX_DIMS)
sprintf(dimstr, "%dx%dx%dx%d", dim_len[t][0], dim_len[t][1],
dim_len[t][2], dim_len[t][3]);
else
sprintf(dimstr, "%dx%dx%d", dim_len[t][0], dim_len[t][1],
dim_len[t][2]);
/* Prepare the chunksize string. */
if (storage == NC_CHUNKED)
{
if (ndims[t] == MAX_DIMS)
sprintf(chunkstr, "%dx%dx%dx%d", (int)chunksize[0], (int)chunksize[1],
(int)chunksize[2], (int)chunksize[3]);
else
sprintf(chunkstr, "%dx%dx%d", (int)chunksize[0], (int)chunksize[1],
(int)chunksize[2]);
}
else
strcpy(chunkstr, "contig ");
/* Output results. */
printf("%s\t%s\t%s\t%d\t\t%d\t\t%d\t\t%d\t\t%d\t\t%d\t\t%d\n",
dimstr, chunkstr, mode_name[t], num_files[t], (int)(cache_size[t]/1024),
num_vars[t], mem_used2, open_us, close_us, create_us);
}
}
SUMMARIZE_ERR;
printf("Test for memory consumption...\n");
{
#define NUM_TRIES_100 100
int ncid, i;
int mem_used, mem_used1, mem_used2;
get_mem_used2(&mem_used);
mem_used1 = mem_used;
mem_used2 = mem_used;
printf("start: memuse= %d\t%d\t%d \n",mem_used, mem_used1,
mem_used2);
printf("bef_open\taft_open\taft_close\tused_open\tused_closed\n");
for (i=0; i < NUM_TRIES_100; i++)
{
/* Open the file. NC_NOWRITE tells netCDF we want read-only access
* to the file.*/
get_mem_used2(&mem_used);
nc_set_chunk_cache(10,10,.5);
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
get_mem_used2(&mem_used1);
/* Close the file, freeing all resources. ???? */
if (nc_close(ncid)) ERR;
get_mem_used2(&mem_used2);
if (mem_used2 - mem_used)
printf("try %d - %d\t\t%d\t\t%d\t\t%d\t\t%d \n", i,
mem_used, mem_used1, mem_used2, mem_used1 - mem_used,
mem_used2 - mem_used);
}
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}
