/**
* Test the darray functionality. Create a netCDF file with 4
* dimensions and 1 PIO_INT variable, and use darray to write some
* data.
*
* @param iosysid the IO system ID.
* @param ioid the ID of the decomposition.
* @param num_flavors the number of IOTYPES available in this build.
* @param flavor array of available iotypes.
* @param my_rank rank of this task.
* @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray().
* @returns 0 for success, error code otherwise.
*/
int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
int provide_fill)
{
char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
int dimids[NDIM]; /* The dimension IDs. */
int ncid; /* The ncid of the netCDF file. */
int ncid2; /* The ncid of the re-opened netCDF file. */
int varid; /* The ID of the netCDF varable. */
int ret; /* Return code. */
PIO_Offset arraylen = 16;
int int_fillvalue = NC_FILL_INT;
void *fillvalue = NULL;
int test_data[arraylen];
int test_data2[arraylen];
int test_data_in[arraylen];
/* Initialize some data. */
for (int f = 0; f < arraylen; f++)
{
test_data[f] = my_rank * 10 + f;
test_data2[f] = 2 * (my_rank * 10 + f);
}
/* Are we providing a fill value? */
if (provide_fill)
fillvalue = &int_fillvalue;
/* Use PIO to create the example file in each of the four
* available ways. */
for (int fmt = 0; fmt < num_flavors; fmt++)
{
/* Create the filename. */
sprintf(filename, "data_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
/* Create the netCDF output file. */
if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
ERR(ret);
/* Turn on fill mode. */
if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
ERR(ret);
/* Define netCDF dimensions and variable. */
for (int d = 0; d < NDIM; d++)
if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
ERR(ret);
/* Define a variable. */
if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid)))
ERR(ret);
/* End define mode. */
if ((ret = PIOc_enddef(ncid)))
ERR(ret);
/* Set the value of the record dimension. */
if ((ret = PIOc_setframe(ncid, varid, 0)))
ERR(ret);
/* Write the data. */
if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
ERR(ret);
/* Set the value of the record dimension to the second record. */
if ((ret = PIOc_setframe(ncid, varid, 1)))
ERR(ret);
/* Write the data for the second record. */
if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data2, fillvalue)))
ERR(ret);
/* Close the netCDF file. */
if ((ret = PIOc_closefile(ncid)))
ERR(ret);
/* Reopen the file. */
if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
ERR(ret);
/* Set the value of the record dimension. */
if ((ret = PIOc_setframe(ncid2, varid, 0)))
ERR(ret);
/* Read the data. */
if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
ERR(ret);
/* Check the results. */
for (int f = 0; f < arraylen; f++)
if (test_data_in[f] != test_data[f])
return ERR_WRONG;
/* Set the value of the record dimension to the second record. */
if ((ret = PIOc_setframe(ncid2, varid, 1)))
ERR(ret);
/* Read the data. */
if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
ERR(ret);
/* Check the results. */
for (int f = 0; f < arraylen; f++)
if (test_data_in[f] != test_data2[f])
return ERR_WRONG;
/* Close the netCDF file. */
if ((ret = PIOc_closefile(ncid2)))
ERR(ret);
}
return PIO_NOERR;
}
/** Main execution of code.
Executes the functions to:
- create a new examplePioClass instance
- initialize MPI and the ParallelIO libraries
- create the decomposition for this example
- create the netCDF output file
- define the variable in the file
- write data to the variable in the file using decomposition
- read the data back from the file using decomposition
- close the file
- clean up resources
The example can be run from the command line (on system that support it) like this:
<pre>
mpiexec -n 4 ./examplePio
</pre>
The sample file created by this program is a small netCDF file. It
has the following contents (as shown by ncdump) for a 4-processor
run:
<pre>
netcdf examplePio_c {
dimensions:
x = 16 ;
variables:
int foo(x) ;
data:
foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ;
}
</pre>
@param [in] argc argument count (should be zero)
@param [in] argv argument array (should be NULL)
@retval examplePioClass* Pointer to self.
*/
int main(int argc, char* argv[])
{
/** Set to non-zero to get output to stdout. */
int verbose = 0;
/** Zero-based rank of processor. */
int my_rank;
/** Number of processors involved in current execution. */
int ntasks;
/** Different output flavors. The example file is written (and
* then read) four times. The first two flavors,
* parallel-netcdf, and netCDF serial, both produce a netCDF
* classic format file (but with different libraries). The
* last two produce netCDF4/HDF5 format files, written with
* and without using netCDF-4 parallel I/O. */
int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF,
PIO_IOTYPE_NETCDF,
PIO_IOTYPE_NETCDF4C,
PIO_IOTYPE_NETCDF4P};
/** Names for the output files. Two of them (pnetcdf and
* classic) will be in classic netCDF format, the others
* (serial4 and parallel4) will be in netCDF-4/HDF5
* format. All four can be read by the netCDF library, and all
* will contain the same contents. */
char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"example2_pnetcdf.nc",
"example2_classic.nc",
"example2_serial4.nc",
"example2_parallel4.nc"};
/** Number of processors that will do IO. In this example we
* will do IO from all processors. */
int niotasks;
/** Stride in the mpi rank between io tasks. Always 1 in this
* example. */
int ioproc_stride = 1;
/** Number of the aggregator? Always 0 in this example. */
int numAggregator = 0;
/** Zero based rank of first processor to be used for I/O. */
int ioproc_start = 0;
/** Specifies the flavor of netCDF output format. */
int iotype;
/** The dimension IDs. */
int dimids[NDIM];
/** Array index per processing unit. This is the number of
* elements of the data array that will be handled by each
* processor. In this example there are 16 data elements. If the
* example is run on 4 processors, then arrIdxPerPe will be 4. */
PIO_Offset elements_per_pe;
/** The ID for the parallel I/O system. It is set by
* PIOc_Init_Intracomm(). It references an internal structure
* containing the general IO subsystem data and MPI
* structure. It is passed to PIOc_finalize() to free
* associated resources, after all I/O, but before
* MPI_Finalize is called. */
int iosysid;
/** The ncid of the netCDF file created in this example. */
int ncid = 0;
/** The ID of the netCDF varable in the example file. */
int varid;
/** The I/O description ID as passed back by PIOc_InitDecomp()
* and freed in PIOc_freedecomp(). */
int ioid;
/** A buffer for sample data. The size of this array will
* vary depending on how many processors are involved in the
* execution of the example code. It's length will be the same
* as elements_per_pe.*/
float *buffer;
/** A buffer for reading data back from the file. The size of
* this array will vary depending on how many processors are
* involved in the execution of the example code. It's length
* will be the same as elements_per_pe.*/
int *read_buffer;
/** A 1-D array which holds the decomposition mapping for this
* example. The size of this array will vary depending on how
* many processors are involved in the execution of the
* example code. It's length will be the same as
* elements_per_pe. */
PIO_Offset *compdof;
#ifdef HAVE_MPE
/** MPE event numbers used to track start and stop of
* different parts of the program for later display with
* Jumpshot. */
int event_num[2][NUM_EVENTS];
#endif /* HAVE_MPE */
/** Needed for command line processing. */
int c;
/* Parse command line. */
while ((c = getopt(argc, argv, "v")) != -1)
switch (c)
{
case 'v':
verbose++;
break;
default:
break;
}
#ifdef TIMING
/* Initialize the GPTL timing library. */
int ret;
if ((ret = GPTLinitialize ()))
return ret;
#endif
/* Initialize MPI. */
if ((ret = MPI_Init(&argc, &argv)))
MPIERR(ret);
if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
MPIERR(ret);
/* Learn my rank and the total number of processors. */
if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
MPIERR(ret);
if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
MPIERR(ret);
/* Check that a valid number of processors was specified. */
if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 ||
ntasks == 8 || ntasks == 16))
fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n");
if (verbose)
printf("%d: ParallelIO Library example1 running on %d processors.\n",
my_rank, ntasks);
#ifdef HAVE_MPE
/* Initialize MPE logging. */
if ((ret = MPE_Init_log()))
ERR(ret);
if (init_logging(my_rank, event_num))
ERR(ERR_LOGGING);
/* Log with MPE that we are starting INIT. */
if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* keep things simple - 1 iotask per MPI process */
niotasks = ntasks;
/* Initialize the PIO IO system. This specifies how
* many and which processors are involved in I/O. */
if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride,
ioproc_start, PIO_REARR_SUBSET, &iosysid)))
ERR(ret);
/* Describe the decomposition. This is a 1-based array, so add 1! */
elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks;
if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
return PIO_ENOMEM;
for (int i = 0; i < elements_per_pe; i++) {
compdof[i] = my_rank * elements_per_pe + i + 1;
}
/* Create the PIO decomposition for this example. */
if (verbose)
printf("rank: %d Creating decomposition...\n", my_rank);
if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe,
compdof, &ioid, NULL, NULL, NULL)))
ERR(ret);
free(compdof);
#ifdef HAVE_MPE
/* Log with MPE that we are done with INIT. */
if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Use PIO to create the example file in each of the four
* available ways. */
for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++)
{
#ifdef HAVE_MPE
/* Log with MPE that we are starting CREATE. */
if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Create the netCDF output file. */
if (verbose)
printf("rank: %d Creating sample file %s with format %d...\n",
my_rank, filename[fmt], format[fmt]);
if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt],
PIO_CLOBBER)))
ERR(ret);
/* Define netCDF dimensions and variable. */
if (verbose)
printf("rank: %d Defining netCDF metadata...\n", my_rank);
for (int d = 0; d < NDIM; d++) {
if (verbose)
printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank,
dim_name[d], dim_len[d]);
if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
ERR(ret);
}
if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
ERR(ret);
/* For netCDF-4 files, set the chunksize to improve performance. */
if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)))
ERR(ret);
if ((ret = PIOc_enddef(ncid)))
ERR(ret);
#ifdef HAVE_MPE
/* Log with MPE that we are done with CREATE. */
if ((ret = MPE_Log_event(event_num[END][CREATE_PNETCDF + fmt], 0, "end create")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Allocate space for sample data. */
if (!(buffer = malloc(elements_per_pe * sizeof(float))))
return PIO_ENOMEM;
/* Write data for each timestep. */
for (int ts = 0; ts < NUM_TIMESTEPS; ts++) {
#ifdef HAVE_MPE
/* Log with MPE that we are starting CALCULATE. */
if ((ret = MPE_Log_event(event_num[START][CALCULATE], 0, "start calculate")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Calculate sample data. Add some math function calls to make this slower. */
for (int i = 0; i < elements_per_pe; i++)
if ((ret = calculate_value(my_rank, ts, &buffer[i])))
ERR(ret);
#ifdef HAVE_MPE
/* Log with MPE that we are done with CALCULATE. */
if ((ret = MPE_Log_event(event_num[END][CALCULATE], 0, "end calculate")))
MPIERR(ret);
/* Log with MPE that we are starting WRITE. */
if ((ret = MPE_Log_event(event_num[START][WRITE], 0, "start write")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Write data to the file. */
if (verbose)
printf("rank: %d Writing sample data...\n", my_rank);
if ((ret = PIOc_setframe(ncid, varid, ts)))
ERR(ret);
if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe,
buffer, NULL)))
ERR(ret);
if ((ret = PIOc_sync(ncid)))
ERR(ret);
#ifdef HAVE_MPE
/* Log with MPE that we are done with WRITE. */
if ((ret = MPE_Log_event(event_num[END][WRITE], 0, "end write")))
MPIERR(ret);
#endif /* HAVE_MPE */
}
#ifdef HAVE_MPE
/* Log with MPE that we are starting CLOSE. */
if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start close")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Free buffer space used in this example. */
free(buffer);
/* Close the netCDF file. */
if (verbose)
printf("rank: %d Closing the sample data file...\n", my_rank);
if ((ret = PIOc_closefile(ncid)))
ERR(ret);
#ifdef HAVE_MPE
/* Log with MPE that we are done with CLOSE. */
if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end close")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* After each file is closed, make all processors wait so that
* all start creating the next file at the same time. */
if ((ret = MPI_Barrier(MPI_COMM_WORLD)))
MPIERR(ret);
}
#ifdef HAVE_MPE
/* Log with MPE that we are starting FREE. */
if ((ret = MPE_Log_event(event_num[START][FREE], 0, "start free")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Free the PIO decomposition. */
if (verbose)
printf("rank: %d Freeing PIO decomposition...\n", my_rank);
if ((ret = PIOc_freedecomp(iosysid, ioid)))
ERR(ret);
/* Finalize the IO system. */
if (verbose)
printf("rank: %d Freeing PIO resources...\n", my_rank);
if ((ret = PIOc_finalize(iosysid)))
ERR(ret);
#ifdef HAVE_MPE
/* Log with MPE that we are done with FREE. */
if ((ret = MPE_Log_event(event_num[END][FREE], 0, "end free")))
MPIERR(ret);
/* Log with MPE that we are starting READ. */
if ((ret = MPE_Log_event(event_num[START][READ], 0, "start read")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Check the output file. */
/* if (!my_rank) */
/* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */
/* if ((ret = check_file(ntasks, filename[fmt]))) */
/* ERR(ret); */
#ifdef HAVE_MPE
/* Log with MPE that we are done with READ. */
if ((ret = MPE_Log_event(event_num[END][READ], 0, "end read")))
MPIERR(ret);
#endif /* HAVE_MPE */
/* Finalize the MPI library. */
MPI_Finalize();
#ifdef TIMING
/* Finalize the GPTL timing library. */
if ((ret = GPTLfinalize ()))
return ret;
#endif
if (verbose)
printf("rank: %d SUCCESS!\n", my_rank);
return 0;
}
/* Run tests for darray functions. */
int main(int argc, char **argv)
{
int my_rank;
int ntasks;
int num_flavors; /* Number of PIO netCDF flavors in this build. */
int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
MPI_Comm test_comm; /* A communicator for this test. */
int ret; /* Return code. */
/* Initialize test. */
if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
MIN_NTASKS, -1, &test_comm)))
ERR(ERR_INIT);
if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
return ret;
/* Only do something on max_ntasks tasks. */
if (my_rank < TARGET_NTASKS)
{
int iosysid; /* The ID for the parallel I/O system. */
int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */
int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */
int wioid, rioid;
int maplen = MAPLEN;
MPI_Offset wcompmap[MAPLEN];
MPI_Offset rcompmap[MAPLEN];
int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
/* Data we will write for each type. */
signed char byte_data[MAPLEN];
char char_data[MAPLEN];
short short_data[MAPLEN];
int int_data[MAPLEN];
float float_data[MAPLEN];
double double_data[MAPLEN];
#ifdef _NETCDF4
unsigned char ubyte_data[MAPLEN];
unsigned short ushort_data[MAPLEN];
unsigned int uint_data[MAPLEN];
long long int64_data[MAPLEN];
unsigned long long uint64_data[MAPLEN];
#endif /* _NETCDF4 */
/* Expected results for each type. */
signed char byte_expected[MAPLEN];
char char_expected[MAPLEN];
short short_expected[MAPLEN];
int int_expected[MAPLEN];
float float_expected[MAPLEN];
double double_expected[MAPLEN];
#ifdef _NETCDF4
unsigned char ubyte_expected[MAPLEN];
unsigned short ushort_expected[MAPLEN];
unsigned int uint_expected[MAPLEN];
long long int64_expected[MAPLEN];
unsigned long long uint64_expected[MAPLEN];
#endif /* _NETCDF4 */
/* Custom fill value for each type. */
signed char byte_fill = -2;
char char_fill = 2;
short short_fill = -2;
int int_fill = -2;
float float_fill = -2;
double double_fill = -2;
#ifdef _NETCDF4
unsigned char ubyte_fill = 2;
unsigned short ushort_fill = 2;
unsigned int uint_fill = 2;
long long int64_fill = 2;
unsigned long long uint64_fill = 2;
#endif /* _NETCDF4 */
/* Default fill value for each type. */
signed char byte_default_fill = NC_FILL_BYTE;
char char_default_fill = NC_FILL_CHAR;
short short_default_fill = NC_FILL_SHORT;
int int_default_fill = NC_FILL_INT;
float float_default_fill = NC_FILL_FLOAT;
double double_default_fill = NC_FILL_DOUBLE;
#ifdef _NETCDF4
unsigned char ubyte_default_fill = NC_FILL_UBYTE;
unsigned short ushort_default_fill = NC_FILL_USHORT;
unsigned int uint_default_fill = NC_FILL_UINT;
long long int64_default_fill = NC_FILL_INT64;
unsigned long long uint64_default_fill = NC_FILL_UINT64;
#endif /* _NETCDF4 */
int ret; /* Return code. */
/* Set up the compmaps. Don't forget these are 1-based
* numbers, like in Fortran! */
for (int i = 0; i < MAPLEN; i++)
{
wcompmap[i] = i % 2 ? my_rank * MAPLEN + i + 1 : 0; /* Even values missing. */
rcompmap[i] = my_rank * MAPLEN + i + 1;
}
/* Figure out iotypes. */
if ((ret = get_iotypes(&num_flavors, flavor)))
ERR(ret);
/* Test for each rearranger. */
for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
{
/* Initialize the PIO IO system. This specifies how
* many and which processors are involved in I/O. */
if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO_PROCS, ioproc_stride, ioproc_start,
rearranger[r], &iosysid)))
return ret;
/* Test with and without custom fill values. */
for (int fv = 0; fv < NUM_TEST_CASES_FILLVALUE; fv++)
{
#ifndef _NETCDF4
#define NUM_TYPES 6
int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
#else
#define NUM_TYPES 11
int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
#endif /* _NETCDF4 */
/* Determine what data to write. Put value of 42 into
* array elements that will not get written. Due to
* the decomposition, these will be replaced by fill
* values. */
for (int i = 0; i < MAPLEN; i++)
{
byte_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
char_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
short_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
int_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
float_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
double_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
#ifdef _NETCDF4
ubyte_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
ushort_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
uint_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
int64_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
uint64_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
#endif /* _NETCDF4 */
}
/* Determine what data to expect from the test. For
* even values of i, the fill value will be used, and
* it may be custom or default fill value. */
for (int i = 0; i < MAPLEN; i++)
{
byte_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? byte_default_fill : byte_fill);
char_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? char_default_fill : char_fill);
short_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? short_default_fill : short_fill);
int_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? int_default_fill : int_fill);
float_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? float_default_fill : float_fill);
double_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? double_default_fill : double_fill);
#ifdef _NETCDF4
ubyte_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? ubyte_default_fill : ubyte_fill);
ushort_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? ushort_default_fill : ushort_fill);
uint_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? uint_default_fill : uint_fill);
int64_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? int64_default_fill : int64_fill);
uint64_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? uint64_default_fill : uint64_fill);
#endif /* _NETCDF4 */
}
/* Test for each available type. */
for (int t = 0; t < NUM_TYPES; t++)
{
void *expected;
void *fill;
void *data;
int ncid, dimid, varid;
char filename[NC_MAX_NAME + 1];
switch (test_type[t])
{
case PIO_BYTE:
expected = byte_expected;
fill = fv ? &byte_default_fill : &byte_fill;
data = byte_data;
break;
case PIO_CHAR:
expected = char_expected;
fill = fv ? &char_default_fill : &char_fill;
data = char_data;
break;
case PIO_SHORT:
expected = short_expected;
fill = fv ? &short_default_fill : &short_fill;
data = short_data;
break;
case PIO_INT:
expected = int_expected;
fill = fv ? &int_default_fill : &int_fill;
data = int_data;
break;
case PIO_FLOAT:
expected = float_expected;
fill = fv ? &float_default_fill : &float_fill;
data = float_data;
break;
case PIO_DOUBLE:
expected = double_expected;
fill = fv ? &double_default_fill : &double_fill;
data = double_data;
break;
#ifdef _NETCDF4
case PIO_UBYTE:
expected = ubyte_expected;
fill = fv ? &ubyte_default_fill : &ubyte_fill;
data = ubyte_data;
break;
case PIO_USHORT:
expected = ushort_expected;
fill = fv ? &ushort_default_fill : &ushort_fill;
data = ushort_data;
break;
case PIO_UINT:
expected = uint_expected;
fill = fv ? &uint_default_fill : &uint_fill;
data = uint_data;
break;
case PIO_INT64:
expected = int64_expected;
fill = fv ? &int64_default_fill : &int64_fill;
data = int64_data;
break;
case PIO_UINT64:
expected = uint64_expected;
fill = fv ? &uint64_default_fill : &uint64_fill;
data = uint64_data;
break;
#endif /* _NETCDF4 */
default:
return ERR_AWFUL;
}
/* Initialize decompositions. */
if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, wcompmap,
&wioid, &rearranger[r], NULL, NULL)))
return ret;
if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, rcompmap,
&rioid, &rearranger[r], NULL, NULL)))
return ret;
/* Create the test file in each of the available iotypes. */
for (int fmt = 0; fmt < num_flavors; fmt++)
{
PIO_Offset type_size;
void *data_in;
/* Byte type doesn't work with pnetcdf. */
if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (test_type[t] == PIO_BYTE || test_type[t] == PIO_CHAR))
continue;
/* NetCDF-4 types only work with netCDF-4 formats. */
if (test_type[t] > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
flavor[fmt] != PIO_IOTYPE_NETCDF4P)
continue;
/* Put together filename. */
sprintf(filename, "%s_iotype_%d_rearr_%d_type_%d.nc", TEST_NAME, flavor[fmt],
rearranger[r], test_type[t]);
/* Create file. */
if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER)))
return ret;
/* Define metadata. */
if ((ret = PIOc_def_dim(ncid, DIM_NAME, dim_len[0], &dimid)))
return ret;
if ((ret = PIOc_def_var(ncid, VAR_NAME, test_type[t], NDIM1, &dimid, &varid)))
return ret;
if ((ret = PIOc_put_att(ncid, varid, FILL_VALUE_NAME, test_type[t],
1, fill)))
return ret;
if ((ret = PIOc_enddef(ncid)))
return ret;
/* Write some data. */
if ((ret = PIOc_write_darray(ncid, varid, wioid, MAPLEN, data, fill)))
return ret;
if ((ret = PIOc_sync(ncid)))
return ret;
/* What is size of type? */
if ((ret = PIOc_inq_type(ncid, test_type[t], NULL, &type_size)))
return ret;
/* Allocate space to read data into. */
if (!(data_in = malloc(type_size * MAPLEN)))
return PIO_ENOMEM;
/* Read the data. */
if ((ret = PIOc_read_darray(ncid, varid, rioid, MAPLEN, data_in)))
return ret;
/* Check results. */
if (memcmp(data_in, expected, type_size * MAPLEN))
return ERR_AWFUL;
/* Release storage. */
free(data_in);
/* Close file. */
if ((ret = PIOc_closefile(ncid)))
return ret;
} /* next iotype */
/* Free decompositions. */
if ((ret = PIOc_freedecomp(iosysid, wioid)))
return ret;
if ((ret = PIOc_freedecomp(iosysid, rioid)))
return ret;
} /* next type */
} /* next fill value test case */
} /* next rearranger */
/* Finalize PIO system. */
if ((ret = PIOc_finalize(iosysid)))
return ret;
} /* endif my_rank < TARGET_NTASKS */
/* Finalize the MPI library. */
if ((ret = pio_test_finalize(&test_comm)))
return ret;
printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
return 0;
}
/** @brief Main execution of code.
Executes the functions to:
- create a new examplePioClass instance
- initialize MPI and the ParallelIO libraries
- create the decomposition for this example
- create the netCDF output file
- define the variable in the file
- write data to the variable in the file using decomposition
- read the data back from the file using decomposition
- close the file
- clean up resources
The example can be run from the command line (on system that support it) like this:
<pre>
mpiexec -n 4 ./examplePio
</pre>
The sample file created by this program is a small netCDF file. It
has the following contents (as shown by ncdump) for a 4-processor
run:
<pre>
netcdf examplePio_c {
dimensions:
x = 16 ;
variables:
int foo(x) ;
data:
foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ;
}
</pre>
@param [in] argc argument count (should be zero)
@param [in] argv argument array (should be NULL)
@retval examplePioClass* Pointer to self.
*/
int main(int argc, char* argv[])
{
/** Set to non-zero to get output to stdout. */
int verbose = 0;
/** Zero-based rank of processor. */
int my_rank;
/** Number of processors involved in current execution. */
int ntasks;
/** Different output flavors. The example file is written (and
* then read) four times. The first two flavors,
* parallel-netcdf, and netCDF serial, both produce a netCDF
* classic format file (but with different libraries). The
* last two produce netCDF4/HDF5 format files, written with
* and without using netCDF-4 parallel I/O. */
int format[NUM_NETCDF_FLAVORS];
/** Number of processors that will do IO. In this example we
* will do IO from all processors. */
int niotasks;
/** Stride in the mpi rank between io tasks. Always 1 in this
* example. */
int ioproc_stride = 1;
/** Zero based rank of first processor to be used for I/O. */
int ioproc_start = 0;
/** The dimension ID. */
int dimid;
/** Array index per processing unit. This is the number of
* elements of the data array that will be handled by each
* processor. In this example there are 16 data elements. If the
* example is run on 4 processors, then arrIdxPerPe will be 4. */
PIO_Offset elements_per_pe;
/* Length of the dimensions in the data. This simple example
* uses one-dimensional data. The lenght along that dimension
* is DIM_LEN (16). */
int dim_len[1] = {DIM_LEN};
/** The ID for the parallel I/O system. It is set by
* PIOc_Init_Intracomm(). It references an internal structure
* containing the general IO subsystem data and MPI
* structure. It is passed to PIOc_finalize() to free
* associated resources, after all I/O, but before
* MPI_Finalize is called. */
int iosysid;
/** The ncid of the netCDF file created in this example. */
int ncid;
/** The ID of the netCDF varable in the example file. */
int varid;
/** The I/O description ID as passed back by PIOc_InitDecomp()
* and freed in PIOc_freedecomp(). */
int ioid;
/** A buffer for sample data. The size of this array will
* vary depending on how many processors are involved in the
* execution of the example code. It's length will be the same
* as elements_per_pe.*/
int *buffer;
/** A 1-D array which holds the decomposition mapping for this
* example. The size of this array will vary depending on how
* many processors are involved in the execution of the
* example code. It's length will be the same as
* elements_per_pe. */
PIO_Offset *compdof;
/** Test filename. */
char filename[NC_MAX_NAME + 1];
/** The number of netCDF flavors available in this build. */
int num_flavors = 0;
/** Used for command line processing. */
int c;
/** Return value. */
int ret;
/* Parse command line. */
while ((c = getopt(argc, argv, "v")) != -1)
switch (c)
{
case 'v':
verbose++;
break;
default:
break;
}
#ifdef TIMING
/* Initialize the GPTL timing library. */
if ((ret = GPTLinitialize ()))
return ret;
#endif
/* Initialize MPI. */
if ((ret = MPI_Init(&argc, &argv)))
MPIERR(ret);
if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
MPIERR(ret);
/* Learn my rank and the total number of processors. */
if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
MPIERR(ret);
if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
MPIERR(ret);
/* Check that a valid number of processors was specified. */
if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 ||
ntasks == 8 || ntasks == 16))
fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n");
if (verbose)
printf("%d: ParallelIO Library example1 running on %d processors.\n",
my_rank, ntasks);
/* keep things simple - 1 iotask per MPI process */
niotasks = ntasks;
/* Turn on logging if available. */
/* PIOc_set_log_level(4); */
/* Change error handling to return errors. */
if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
return ret;
/* Initialize the PIO IO system. This specifies how
* many and which processors are involved in I/O. */
if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride,
ioproc_start, PIO_REARR_SUBSET, &iosysid)))
ERR(ret);
/* Describe the decomposition. This is a 1-based array, so add 1! */
elements_per_pe = DIM_LEN / ntasks;
if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
return PIO_ENOMEM;
for (int i = 0; i < elements_per_pe; i++)
compdof[i] = my_rank * elements_per_pe + i + 1;
/* Create the PIO decomposition for this example. */
if (verbose)
printf("rank: %d Creating decomposition...\n", my_rank);
if ((ret = PIOc_InitDecomp(iosysid, PIO_INT, NDIM, dim_len, (PIO_Offset)elements_per_pe,
compdof, &ioid, NULL, NULL, NULL)))
ERR(ret);
free(compdof);
/* The number of favors may change with the build parameters. */
#ifdef _PNETCDF
format[num_flavors++] = PIO_IOTYPE_PNETCDF;
#endif
format[num_flavors++] = PIO_IOTYPE_NETCDF;
#ifdef _NETCDF4
format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
#endif
/* Use PIO to create the example file in each of the four
* available ways. */
for (int fmt = 0; fmt < num_flavors; fmt++)
{
/* Create a filename. */
sprintf(filename, "example1_%d.nc", fmt);
/* Create the netCDF output file. */
if (verbose)
printf("rank: %d Creating sample file %s with format %d...\n",
my_rank, filename, format[fmt]);
if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename,
PIO_CLOBBER)))
ERR(ret);
/* Define netCDF dimension and variable. */
if (verbose)
printf("rank: %d Defining netCDF metadata...\n", my_rank);
if ((ret = PIOc_def_dim(ncid, DIM_NAME, (PIO_Offset)dim_len[0], &dimid)))
ERR(ret);
if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, &dimid, &varid)))
ERR(ret);
if ((ret = PIOc_enddef(ncid)))
ERR(ret);
/* Prepare sample data. */
if (!(buffer = malloc(elements_per_pe * sizeof(int))))
return PIO_ENOMEM;
for (int i = 0; i < elements_per_pe; i++)
buffer[i] = START_DATA_VAL + my_rank;
/* Write data to the file. */
if (verbose)
printf("rank: %d Writing sample data...\n", my_rank);
if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe,
buffer, NULL)))
ERR(ret);
if ((ret = PIOc_sync(ncid)))
ERR(ret);
/* Free buffer space used in this example. */
free(buffer);
/* Close the netCDF file. */
if (verbose)
printf("rank: %d Closing the sample data file...\n", my_rank);
if ((ret = PIOc_closefile(ncid)))
ERR(ret);
}
/* Free the PIO decomposition. */
if (verbose)
printf("rank: %d Freeing PIO decomposition...\n", my_rank);
if ((ret = PIOc_freedecomp(iosysid, ioid)))
ERR(ret);
/* Finalize the IO system. */
if (verbose)
printf("rank: %d Freeing PIO resources...\n", my_rank);
if ((ret = PIOc_finalize(iosysid)))
ERR(ret);
/* Check the output file. */
if (!my_rank)
for (int fmt = 0; fmt < num_flavors; fmt++)
{
sprintf(filename, "example1_%d.nc", fmt);
if ((ret = check_file(ntasks, filename)))
ERR(ret);
}
/* Finalize the MPI library. */
MPI_Finalize();
#ifdef TIMING
/* Finalize the GPTL timing library. */
if ((ret = GPTLfinalize ()))
return ret;
#endif
if (verbose)
printf("rank: %d SUCCESS!\n", my_rank);
return 0;
}
/* Write, then read, a simple example with darrays.
The sample file created by this program is a small netCDF file. It
has the following contents (as shown by ncdump):
<pre>
netcdf darray_no_async_iotype_1 {
dimensions:
unlimted = UNLIMITED ; // (2 currently)
x = 4 ;
y = 4 ;
variables:
int foo(unlimted, x, y) ;
data:
foo =
42, 42, 42, 42,
43, 43, 43, 43,
44, 44, 44, 44,
45, 45, 45, 45,
142, 142, 142, 142,
143, 143, 143, 143,
144, 144, 144, 144,
145, 145, 145, 145 ;
}
</pre>
*/
int main(int argc, char* argv[])
{
int my_rank; /* Zero-based rank of processor. */
int ntasks; /* Number of processors involved in current execution. */
int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */
int ioproc_start = 0; /* Rank of first task to be used for I/O. */
PIO_Offset elements_per_pe; /* Array elements per processing unit. */
int iosysid; /* The ID for the parallel I/O system. */
int ncid; /* The ncid of the netCDF file. */
int dimid[NDIM3]; /* The dimension ID. */
int varid; /* The ID of the netCDF varable. */
int ioid; /* The I/O description ID. */
char filename[NC_MAX_NAME + 1]; /* Test filename. */
int num_flavors = 0; /* Number of iotypes available in this build. */
int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */
int ret; /* Return value. */
#ifdef TIMING
/* Initialize the GPTL timing library. */
if ((ret = GPTLinitialize ()))
return ret;
#endif
/* Initialize MPI. */
if ((ret = MPI_Init(&argc, &argv)))
MPIERR(ret);
if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
MPIERR(ret);
/* Learn my rank and the total number of processors. */
if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
MPIERR(ret);
if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
MPIERR(ret);
/* Check that a valid number of processors was specified. */
if (ntasks != TARGET_NTASKS)
fprintf(stderr, "Number of processors must be 4!\n");
printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n",
my_rank, ntasks);
/* Turn on logging. */
if ((ret = PIOc_set_log_level(LOG_LEVEL)))
return ret;
/* Initialize the PIO IO system. This specifies how many and
* which processors are involved in I/O. */
if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride,
ioproc_start, PIO_REARR_BOX, &iosysid)))
ERR(ret);
/* Describe the decomposition. */
elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS;
/* Allocate and initialize array of decomposition mapping. */
PIO_Offset compdof[elements_per_pe];
for (int i = 0; i < elements_per_pe; i++)
compdof[i] = my_rank * elements_per_pe + i;
/* Create the PIO decomposition for this example. Since this
* is a variable with an unlimited dimension, we want to
* create a 2-D composition which represents one record. */
printf("rank: %d Creating decomposition...\n", my_rank);
if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe,
compdof, &ioid, 0, NULL, NULL)))
ERR(ret);
/* The number of favors may change with the build parameters. */
#ifdef _PNETCDF
format[num_flavors++] = PIO_IOTYPE_PNETCDF;
#endif
format[num_flavors++] = PIO_IOTYPE_NETCDF;
#ifdef _NETCDF4
format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
#endif
/* Use PIO to create the example file in each of the four
* available ways. */
for (int fmt = 0; fmt < num_flavors; fmt++)
{
/* Create a filename. */
sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]);
/* Create the netCDF output file. */
printf("rank: %d Creating sample file %s with format %d...\n",
my_rank, filename, format[fmt]);
if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER)))
ERR(ret);
/* Define netCDF dimension and variable. */
printf("rank: %d Defining netCDF metadata...\n", my_rank);
for (int d = 0; d < NDIM3; d++)
if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
ERR(ret);
if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid)))
ERR(ret);
if ((ret = PIOc_enddef(ncid)))
ERR(ret);
/* Allocate storage for sample data. */
int buffer[elements_per_pe];
/* Write each timestep. */
for (int t = 0; t < NUM_TIMESTEPS; t++)
{
/* Create some data for this timestep. */
for (int i = 0; i < elements_per_pe; i++)
buffer[i] = 100 * t + START_DATA_VAL + my_rank;
/* Write data to the file. */
printf("rank: %d Writing sample data...\n", my_rank);
if ((ret = PIOc_setframe(ncid, varid, t)))
ERR(ret);
if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL)))
ERR(ret);
}
/* THis will cause all data to be written to disk. */
if ((ret = PIOc_sync(ncid)))
ERR(ret);
/* Close the netCDF file. */
printf("rank: %d Closing the sample data file...\n", my_rank);
if ((ret = PIOc_closefile(ncid)))
ERR(ret);
/* Check the output file. */
/* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */
/* my_rank, ioid))) */
/* ERR(ret); */
}
/* Free the PIO decomposition. */
printf("rank: %d Freeing PIO decomposition...\n", my_rank);
if ((ret = PIOc_freedecomp(iosysid, ioid)))
ERR(ret);
/* Finalize the IO system. */
printf("rank: %d Freeing PIO resources...\n", my_rank);
if ((ret = PIOc_finalize(iosysid)))
ERR(ret);
/* Finalize the MPI library. */
MPI_Finalize();
#ifdef TIMING
/* Finalize the GPTL timing library. */
if ((ret = GPTLfinalize ()))
return ret;
#endif
printf("rank: %d SUCCESS!\n", my_rank);
return 0;
}
