/** * Test the darray functionality. Create a netCDF file with 4 * dimensions and 1 PIO_INT variable, and use darray to write some * data. * * @param iosysid the IO system ID. * @param ioid the ID of the decomposition. * @param num_flavors the number of IOTYPES available in this build. * @param flavor array of available iotypes. * @param my_rank rank of this task. * @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray(). * @returns 0 for success, error code otherwise. */ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, int provide_fill) { char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ int dimids[NDIM]; /* The dimension IDs. */ int ncid; /* The ncid of the netCDF file. */ int ncid2; /* The ncid of the re-opened netCDF file. */ int varid; /* The ID of the netCDF varable. */ int ret; /* Return code. */ PIO_Offset arraylen = 16; int int_fillvalue = NC_FILL_INT; void *fillvalue = NULL; int test_data[arraylen]; int test_data2[arraylen]; int test_data_in[arraylen]; /* Initialize some data. */ for (int f = 0; f < arraylen; f++) { test_data[f] = my_rank * 10 + f; test_data2[f] = 2 * (my_rank * 10 + f); } /* Are we providing a fill value? */ if (provide_fill) fillvalue = &int_fillvalue; /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* Create the filename. */ sprintf(filename, "data_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) ERR(ret); /* Turn on fill mode. */ if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL))) ERR(ret); /* Define netCDF dimensions and variable. */ for (int d = 0; d < NDIM; d++) if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); /* Define a variable. */ if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid))) ERR(ret); /* End define mode. */ if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid, varid, 0))) ERR(ret); /* Write the data. */ if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) ERR(ret); /* Set the value of the record dimension to the second record. */ if ((ret = PIOc_setframe(ncid, varid, 1))) ERR(ret); /* Write the data for the second record. */ if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data2, fillvalue))) ERR(ret); /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid))) ERR(ret); /* Reopen the file. */ if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) ERR(ret); /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid2, varid, 0))) ERR(ret); /* Read the data. */ if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in))) ERR(ret); /* Check the results. */ for (int f = 0; f < arraylen; f++) if (test_data_in[f] != test_data[f]) return ERR_WRONG; /* Set the value of the record dimension to the second record. */ if ((ret = PIOc_setframe(ncid2, varid, 1))) ERR(ret); /* Read the data. */ if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in))) ERR(ret); /* Check the results. */ for (int f = 0; f < arraylen; f++) if (test_data_in[f] != test_data2[f]) return ERR_WRONG; /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid2))) ERR(ret); } return PIO_NOERR; }
/** Main execution of code. Executes the functions to: - create a new examplePioClass instance - initialize MPI and the ParallelIO libraries - create the decomposition for this example - create the netCDF output file - define the variable in the file - write data to the variable in the file using decomposition - read the data back from the file using decomposition - close the file - clean up resources The example can be run from the command line (on system that support it) like this: <pre> mpiexec -n 4 ./examplePio </pre> The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump) for a 4-processor run: <pre> netcdf examplePio_c { dimensions: x = 16 ; variables: int foo(x) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ; } </pre> @param [in] argc argument count (should be zero) @param [in] argv argument array (should be NULL) @retval examplePioClass* Pointer to self. */ int main(int argc, char* argv[]) { /** Set to non-zero to get output to stdout. */ int verbose = 0; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Different output flavors. The example file is written (and * then read) four times. The first two flavors, * parallel-netcdf, and netCDF serial, both produce a netCDF * classic format file (but with different libraries). The * last two produce netCDF4/HDF5 format files, written with * and without using netCDF-4 parallel I/O. */ int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P}; /** Names for the output files. Two of them (pnetcdf and * classic) will be in classic netCDF format, the others * (serial4 and parallel4) will be in netCDF-4/HDF5 * format. All four can be read by the netCDF library, and all * will contain the same contents. */ char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"example2_pnetcdf.nc", "example2_classic.nc", "example2_serial4.nc", "example2_parallel4.nc"}; /** Number of processors that will do IO. In this example we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * example. */ int ioproc_stride = 1; /** Number of the aggregator? Always 0 in this example. */ int numAggregator = 0; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** Specifies the flavor of netCDF output format. */ int iotype; /** The dimension IDs. */ int dimids[NDIM]; /** Array index per processing unit. This is the number of * elements of the data array that will be handled by each * processor. In this example there are 16 data elements. If the * example is run on 4 processors, then arrIdxPerPe will be 4. */ PIO_Offset elements_per_pe; /** The ID for the parallel I/O system. It is set by * PIOc_Init_Intracomm(). It references an internal structure * containing the general IO subsystem data and MPI * structure. It is passed to PIOc_finalize() to free * associated resources, after all I/O, but before * MPI_Finalize is called. */ int iosysid; /** The ncid of the netCDF file created in this example. */ int ncid = 0; /** The ID of the netCDF varable in the example file. */ int varid; /** The I/O description ID as passed back by PIOc_InitDecomp() * and freed in PIOc_freedecomp(). */ int ioid; /** A buffer for sample data. The size of this array will * vary depending on how many processors are involved in the * execution of the example code. It's length will be the same * as elements_per_pe.*/ float *buffer; /** A buffer for reading data back from the file. The size of * this array will vary depending on how many processors are * involved in the execution of the example code. It's length * will be the same as elements_per_pe.*/ int *read_buffer; /** A 1-D array which holds the decomposition mapping for this * example. The size of this array will vary depending on how * many processors are involved in the execution of the * example code. It's length will be the same as * elements_per_pe. */ PIO_Offset *compdof; #ifdef HAVE_MPE /** MPE event numbers used to track start and stop of * different parts of the program for later display with * Jumpshot. */ int event_num[2][NUM_EVENTS]; #endif /* HAVE_MPE */ /** Needed for command line processing. */ int c; /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) { case 'v': verbose++; break; default: break; } #ifdef TIMING /* Initialize the GPTL timing library. */ int ret; if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); #ifdef HAVE_MPE /* Initialize MPE logging. */ if ((ret = MPE_Init_log())) ERR(ret); if (init_logging(my_rank, event_num)) ERR(ERR_LOGGING); /* Log with MPE that we are starting INIT. */ if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) { compdof[i] = my_rank * elements_per_pe + i + 1; } /* Create the PIO decomposition for this example. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); #ifdef HAVE_MPE /* Log with MPE that we are done with INIT. */ if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CREATE. */ if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename[fmt], format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], PIO_CLOBBER))) ERR(ret); /* Define netCDF dimensions and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); for (int d = 0; d < NDIM; d++) { if (verbose) printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, dim_name[d], dim_len[d]); if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); } if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); /* For netCDF-4 files, set the chunksize to improve performance. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CREATE. */ if ((ret = MPE_Log_event(event_num[END][CREATE_PNETCDF + fmt], 0, "end create"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Allocate space for sample data. */ if (!(buffer = malloc(elements_per_pe * sizeof(float)))) return PIO_ENOMEM; /* Write data for each timestep. */ for (int ts = 0; ts < NUM_TIMESTEPS; ts++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CALCULATE. */ if ((ret = MPE_Log_event(event_num[START][CALCULATE], 0, "start calculate"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Calculate sample data. Add some math function calls to make this slower. */ for (int i = 0; i < elements_per_pe; i++) if ((ret = calculate_value(my_rank, ts, &buffer[i]))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CALCULATE. */ if ((ret = MPE_Log_event(event_num[END][CALCULATE], 0, "end calculate"))) MPIERR(ret); /* Log with MPE that we are starting WRITE. */ if ((ret = MPE_Log_event(event_num[START][WRITE], 0, "start write"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Write data to the file. */ if (verbose) printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_setframe(ncid, varid, ts))) ERR(ret); if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe, buffer, NULL))) ERR(ret); if ((ret = PIOc_sync(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with WRITE. */ if ((ret = MPE_Log_event(event_num[END][WRITE], 0, "end write"))) MPIERR(ret); #endif /* HAVE_MPE */ } #ifdef HAVE_MPE /* Log with MPE that we are starting CLOSE. */ if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start close"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Free buffer space used in this example. */ free(buffer); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CLOSE. */ if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end close"))) MPIERR(ret); #endif /* HAVE_MPE */ /* After each file is closed, make all processors wait so that * all start creating the next file at the same time. */ if ((ret = MPI_Barrier(MPI_COMM_WORLD))) MPIERR(ret); } #ifdef HAVE_MPE /* Log with MPE that we are starting FREE. */ if ((ret = MPE_Log_event(event_num[START][FREE], 0, "start free"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with FREE. */ if ((ret = MPE_Log_event(event_num[END][FREE], 0, "end free"))) MPIERR(ret); /* Log with MPE that we are starting READ. */ if ((ret = MPE_Log_event(event_num[START][READ], 0, "start read"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Check the output file. */ /* if (!my_rank) */ /* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ /* if ((ret = check_file(ntasks, filename[fmt]))) */ /* ERR(ret); */ #ifdef HAVE_MPE /* Log with MPE that we are done with READ. */ if ((ret = MPE_Log_event(event_num[END][READ], 0, "end read"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif if (verbose) printf("rank: %d SUCCESS!\n", my_rank); return 0; }
/* @brief Check the output file. * * Use netCDF to check that the output is as expected. * * @param ntasks The number of processors running the example. * @param filename The name of the example file to check. * * @return 0 if example file is correct, non-zero otherwise. */ int check_file(int iosysid, int ntasks, char *filename, int iotype, int elements_per_pe, int my_rank, int ioid) { int ncid; /* File ID from netCDF. */ int ndims; /* Number of dimensions. */ int nvars; /* Number of variables. */ int ngatts; /* Number of global attributes. */ int unlimdimid; /* ID of unlimited dimension. */ int natts; /* Number of variable attributes. */ nc_type xtype; /* NetCDF data type of this variable. */ int ret; /* Return code for function calls. */ int dimids[NDIM3]; /* Dimension ids for this variable. */ char var_name[NC_MAX_NAME]; /* Name of the variable. */ /* size_t start[NDIM3]; /\* Zero-based index to start read. *\/ */ /* size_t count[NDIM3]; /\* Number of elements to read. *\/ */ /* int buffer[DIM_LEN_X]; /\* Buffer to read in data. *\/ */ /* int expected[DIM_LEN_X]; /\* Data values we expect to find. *\/ */ /* Open the file. */ if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0))) return ret; printf("opened file %s ncid = %d\n", filename, ncid); /* Check the metadata. */ if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) return ret; /* Check the dimensions. */ if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0) return ERR_BAD; for (int d = 0; d < NDIM3; d++) { char my_dim_name[NC_MAX_NAME]; PIO_Offset dimlen; if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) return ret; if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) return ERR_BAD; } /* Check the variable. */ if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts))) return ret; if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 || dimids[2] != 2 || natts != 0) return ERR_BAD; /* Allocate storage for sample data. */ int buffer[elements_per_pe]; int buffer_in[elements_per_pe]; /* Check each timestep. */ for (int t = 0; t < NUM_TIMESTEPS; t++) { int varid = 0; /* There's only one var in sample file. */ /* This is the data we expect for this timestep. */ for (int i = 0; i < elements_per_pe; i++) buffer[i] = 100 * t + START_DATA_VAL + my_rank; /* Read one record. */ if ((ret = PIOc_setframe(ncid, varid, t))) ERR(ret); if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in))) return ret; /* Check the results. */ for (int i = 0; i < elements_per_pe; i++) if (buffer_in[i] != buffer[i]) return ERR_BAD; } /* Close the file. */ if ((ret = PIOc_closefile(ncid))) return ret; /* Everything looks good! */ return 0; }
/** * Test the darray functionality. Create a netCDF file with 3 * dimensions and 3 variable, and use PIOc_write_darray_multi() to * write one record of data to all three vars at once. * * @param iosysid the IO system ID. * @param ioid the ID of the decomposition. * @param num_flavors the number of IOTYPES available in this build. * @param flavor array of available iotypes. * @param my_rank rank of this task. * @param pio_type the type of the data. * @returns 0 for success, error code otherwise. */ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, int pio_type) { #define NUM_TEST_CASES_WRT_MULTI 2 #define NUM_TEST_CASES_FILLVALUE 2 char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ int dimids[NDIM]; /* The dimension IDs. */ int ncid; /* The ncid of the netCDF file. */ int ncid2; /* The ncid of the re-opened netCDF file. */ int varid[NVAR]; /* The IDs of the netCDF varables. */ int other_varid; /* The IDs of a var of different type. */ int wrong_varid[NVAR]; /* These will not work. */ PIO_Offset arraylen = 4; /* Amount of data from each task. */ void *fillvalue; /* Pointer to fill value. */ void *test_data; /* Pointer to test data we will write. */ void *test_data_in; /* Pointer to buffer we will read into. */ int ret; /* Return code. */ /* Default fill value array for each type. */ signed char byte_fill[NVAR] = {NC_FILL_BYTE, NC_FILL_BYTE, NC_FILL_BYTE}; char char_fill[NVAR] = {NC_FILL_CHAR, NC_FILL_CHAR, NC_FILL_CHAR}; short short_fill[NVAR] = {NC_FILL_SHORT, NC_FILL_SHORT, NC_FILL_SHORT}; int int_fill[NVAR] = {NC_FILL_INT, NC_FILL_INT, NC_FILL_INT}; float float_fill[NVAR] = {NC_FILL_FLOAT, NC_FILL_FLOAT, NC_FILL_FLOAT}; double double_fill[NVAR] = {NC_FILL_DOUBLE, NC_FILL_DOUBLE, NC_FILL_DOUBLE}; #ifdef _NETCDF4 unsigned char ubyte_fill[NVAR] = {NC_FILL_UBYTE, NC_FILL_UBYTE, NC_FILL_UBYTE}; unsigned short ushort_fill[NVAR] = {NC_FILL_USHORT, NC_FILL_USHORT, NC_FILL_USHORT}; unsigned int uint_fill[NVAR] = {NC_FILL_UINT, NC_FILL_UINT, NC_FILL_UINT}; long long int64_fill[NVAR] = {NC_FILL_INT64, NC_FILL_INT64, NC_FILL_INT64}; unsigned long long uint64_fill[NVAR] = {NC_FILL_UINT64, NC_FILL_UINT64, NC_FILL_UINT64}; #endif /* _NETCDF4 */ /* Test data we will write. */ signed char test_data_byte[arraylen * NVAR]; char test_data_char[arraylen * NVAR]; short test_data_short[arraylen * NVAR]; int test_data_int[arraylen * NVAR]; float test_data_float[arraylen * NVAR]; double test_data_double[arraylen * NVAR]; #ifdef _NETCDF4 unsigned char test_data_ubyte[arraylen * NVAR]; unsigned short test_data_ushort[arraylen * NVAR]; unsigned int test_data_uint[arraylen * NVAR]; long long test_data_int64[arraylen * NVAR]; unsigned long long test_data_uint64[arraylen * NVAR]; #endif /* _NETCDF4 */ /* We will read test data into these buffers. */ signed char test_data_byte_in[arraylen]; char test_data_char_in[arraylen]; short test_data_short_in[arraylen]; int test_data_int_in[arraylen]; float test_data_float_in[arraylen]; double test_data_double_in[arraylen]; #ifdef _NETCDF4 unsigned char test_data_ubyte_in[arraylen]; unsigned short test_data_ushort_in[arraylen]; unsigned int test_data_uint_in[arraylen]; long long test_data_int64_in[arraylen]; unsigned long long test_data_uint64_in[arraylen]; #endif /* _NETCDF4 */ /* Initialize a big blob of test data for NVAR vars. */ for (int f = 0; f < arraylen * NVAR; f++) { test_data_byte[f] = my_rank * 1 + f; test_data_char[f] = my_rank * 2 + f; test_data_short[f] = my_rank * 5 + f; test_data_int[f] = my_rank * 10 + f; test_data_float[f] = my_rank * 10 + f + 0.5; test_data_double[f] = my_rank * 100000 + f + 0.5; #ifdef _NETCDF4 test_data_ubyte[f] = my_rank * 3 + f; test_data_ushort[f] = my_rank * 9 + f; test_data_uint[f] = my_rank * 100 + f; test_data_int64[f] = my_rank * 10000 + f; test_data_uint64[f] = my_rank * 100000 + f; #endif /* _NETCDF4 */ } /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* 1-byte types not working with pnetcdf. */ if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) continue; /* NetCDF-4 types only work with netCDF-4. */ if (pio_type > PIO_DOUBLE && (flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P)) continue; /* Add a couple of extra tests for the * PIOc_write_darray_multi() function. */ for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++) { /* Test with/without providing a fill value to PIOc_write_darray(). */ for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) { /* Create the filename. */ sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME, flavor[fmt], pio_type, test_multi, provide_fill); /* Select the fill value and data. */ switch (pio_type) { case PIO_BYTE: fillvalue = provide_fill ? byte_fill : NULL; test_data = test_data_byte; test_data_in = test_data_byte_in; break; case PIO_CHAR: fillvalue = provide_fill ? char_fill : NULL; test_data = test_data_char; test_data_in = test_data_char_in; break; case PIO_SHORT: fillvalue = provide_fill ? short_fill : NULL; test_data = test_data_short; test_data_in = test_data_short_in; break; case PIO_INT: fillvalue = provide_fill ? int_fill : NULL; test_data = test_data_int; test_data_in = test_data_int_in; break; case PIO_FLOAT: fillvalue = provide_fill ? float_fill : NULL; test_data = test_data_float; test_data_in = test_data_float_in; break; case PIO_DOUBLE: fillvalue = provide_fill ? double_fill : NULL; test_data = test_data_double; test_data_in = test_data_double_in; break; #ifdef _NETCDF4 case PIO_UBYTE: fillvalue = provide_fill ? ubyte_fill : NULL; test_data = test_data_ubyte; test_data_in = test_data_ubyte_in; break; case PIO_USHORT: fillvalue = provide_fill ? ushort_fill : NULL; test_data = test_data_ushort; test_data_in = test_data_ushort_in; break; case PIO_UINT: fillvalue = provide_fill ? uint_fill : NULL; test_data = test_data_uint; test_data_in = test_data_uint_in; break; case PIO_INT64: fillvalue = provide_fill ? int64_fill : NULL; test_data = test_data_int64; test_data_in = test_data_int64_in; break; case PIO_UINT64: fillvalue = provide_fill ? uint64_fill : NULL; test_data = test_data_uint64; test_data_in = test_data_uint64_in; break; #endif /* _NETCDF4 */ default: ERR(ERR_WRONG); } /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) ERR(ret); /* Define netCDF dimensions and variable. */ for (int d = 0; d < NDIM; d++) if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); /* Define a variable. */ for (int v = 0; v < NVAR; v++) if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v]))) ERR(ret); /* Define a variable of a different type, to test error handling. */ int other_pio_type = pio_type < 5 ? pio_type + 1 : PIO_INT; if ((ret = PIOc_def_var(ncid, "OTHER_VAR", other_pio_type, NDIM, dimids, &other_varid))) ERR(ret); /* Leave a note. */ if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, NOTE_NAME, strlen(NOTE), NOTE))) ERR(ret); int num_stooges = TOTAL_NUMBER_OF_STOOGES; if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, TOTAL_NUMBER_OF_STOOGES_NAME, PIO_INT, 1, &num_stooges))) ERR(ret); /* End define mode. */ if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid, varid[0], 0))) ERR(ret); int frame[NVAR] = {0, 0, 0}; int flushtodisk = test_multi; /* This will not work, because we mix var types. */ wrong_varid[0] = varid[0]; wrong_varid[1] = varid[1]; wrong_varid[0] = other_varid; // if (PIOc_write_darray_multi(ncid, wrong_varid, ioid, NVAR, arraylen, test_data, frame, // fillvalue, flushtodisk) != PIO_EINVAL) // ERR(ERR_WRONG); /* Write the data with the _multi function. */ if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, NVAR, arraylen, test_data, frame, fillvalue, flushtodisk))) ERR(ret); /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid))) ERR(ret); /* Reopen the file. */ if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) ERR(ret); /* Now use read_darray on each var in turn and make * sure we get correct data. */ for (int v = 0; v < NVAR; v++) { /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid2, varid[v], 0))) ERR(ret); /* Read the data. */ if ((ret = PIOc_read_darray(ncid2, varid[v], ioid, arraylen, test_data_in))) ERR(ret); /* Check the results. */ for (int f = 0; f < arraylen; f++) { switch (pio_type) { case PIO_BYTE: if (test_data_byte_in[f] != test_data_byte[f + arraylen * v]) return ERR_WRONG; break; case PIO_CHAR: if (test_data_char_in[f] != test_data_char[f + arraylen * v]) return ERR_WRONG; break; case PIO_SHORT: if (test_data_short_in[f] != test_data_short[f + arraylen * v]) return ERR_WRONG; break; case PIO_INT: if (test_data_int_in[f] != test_data_int[f + arraylen * v]) return ERR_WRONG; break; case PIO_FLOAT: if (test_data_float_in[f] != test_data_float[f + arraylen * v]) return ERR_WRONG; break; case PIO_DOUBLE: if (test_data_double_in[f] != test_data_double[f + arraylen * v]) return ERR_WRONG; break; #ifdef _NETCDF4 case PIO_UBYTE: if (test_data_ubyte_in[f] != test_data_ubyte[f + arraylen * v]) return ERR_WRONG; break; case PIO_USHORT: if (test_data_ushort_in[f] != test_data_ushort[f + arraylen * v]) return ERR_WRONG; break; case PIO_UINT: if (test_data_uint_in[f] != test_data_uint[f + arraylen * v]) return ERR_WRONG; break; case PIO_INT64: if (test_data_int64_in[f] != test_data_int64[f + arraylen * v]) return ERR_WRONG; break; case PIO_UINT64: if (test_data_uint64_in[f] != test_data_uint64[f + arraylen * v]) return ERR_WRONG; break; #endif /* _NETCDF4 */ default: ERR(ERR_WRONG); } } } /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid2))) ERR(ret); } /* next fillvalue test case */ } /* next test multi */ } /* next iotype */ return PIO_NOERR; }
/* Write, then read, a simple example with darrays. The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump): <pre> netcdf darray_no_async_iotype_1 { dimensions: unlimted = UNLIMITED ; // (2 currently) x = 4 ; y = 4 ; variables: int foo(unlimted, x, y) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45, 142, 142, 142, 142, 143, 143, 143, 143, 144, 144, 144, 144, 145, 145, 145, 145 ; } </pre> */ int main(int argc, char* argv[]) { int my_rank; /* Zero-based rank of processor. */ int ntasks; /* Number of processors involved in current execution. */ int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ int ioproc_start = 0; /* Rank of first task to be used for I/O. */ PIO_Offset elements_per_pe; /* Array elements per processing unit. */ int iosysid; /* The ID for the parallel I/O system. */ int ncid; /* The ncid of the netCDF file. */ int dimid[NDIM3]; /* The dimension ID. */ int varid; /* The ID of the netCDF varable. */ int ioid; /* The I/O description ID. */ char filename[NC_MAX_NAME + 1]; /* Test filename. */ int num_flavors = 0; /* Number of iotypes available in this build. */ int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */ int ret; /* Return value. */ #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (ntasks != TARGET_NTASKS) fprintf(stderr, "Number of processors must be 4!\n"); printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n", my_rank, ntasks); /* Turn on logging. */ if ((ret = PIOc_set_log_level(LOG_LEVEL))) return ret; /* Initialize the PIO IO system. This specifies how many and * which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride, ioproc_start, PIO_REARR_BOX, &iosysid))) ERR(ret); /* Describe the decomposition. */ elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS; /* Allocate and initialize array of decomposition mapping. */ PIO_Offset compdof[elements_per_pe]; for (int i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i; /* Create the PIO decomposition for this example. Since this * is a variable with an unlimited dimension, we want to * create a 2-D composition which represents one record. */ printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, compdof, &ioid, 0, NULL, NULL))) ERR(ret); /* The number of favors may change with the build parameters. */ #ifdef _PNETCDF format[num_flavors++] = PIO_IOTYPE_PNETCDF; #endif format[num_flavors++] = PIO_IOTYPE_NETCDF; #ifdef _NETCDF4 format[num_flavors++] = PIO_IOTYPE_NETCDF4C; format[num_flavors++] = PIO_IOTYPE_NETCDF4P; #endif /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* Create a filename. */ sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); /* Create the netCDF output file. */ printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename, format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) ERR(ret); /* Define netCDF dimension and variable. */ printf("rank: %d Defining netCDF metadata...\n", my_rank); for (int d = 0; d < NDIM3; d++) if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) ERR(ret); if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Allocate storage for sample data. */ int buffer[elements_per_pe]; /* Write each timestep. */ for (int t = 0; t < NUM_TIMESTEPS; t++) { /* Create some data for this timestep. */ for (int i = 0; i < elements_per_pe; i++) buffer[i] = 100 * t + START_DATA_VAL + my_rank; /* Write data to the file. */ printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_setframe(ncid, varid, t))) ERR(ret); if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) ERR(ret); } /* THis will cause all data to be written to disk. */ if ((ret = PIOc_sync(ncid))) ERR(ret); /* Close the netCDF file. */ printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); /* Check the output file. */ /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */ /* my_rank, ioid))) */ /* ERR(ret); */ } /* Free the PIO decomposition. */ printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif printf("rank: %d SUCCESS!\n", my_rank); return 0; }