/** Main execution of code. Executes the functions to: - create a new examplePioClass instance - initialize MPI and the ParallelIO libraries - create the decomposition for this example - create the netCDF output file - define the variable in the file - write data to the variable in the file using decomposition - read the data back from the file using decomposition - close the file - clean up resources The example can be run from the command line (on system that support it) like this: <pre> mpiexec -n 4 ./examplePio </pre> The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump) for a 4-processor run: <pre> netcdf examplePio_c { dimensions: x = 16 ; variables: int foo(x) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ; } </pre> @param [in] argc argument count (should be zero) @param [in] argv argument array (should be NULL) @retval examplePioClass* Pointer to self. */ int main(int argc, char* argv[]) { /** Set to non-zero to get output to stdout. */ int verbose = 0; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Different output flavors. The example file is written (and * then read) four times. The first two flavors, * parallel-netcdf, and netCDF serial, both produce a netCDF * classic format file (but with different libraries). The * last two produce netCDF4/HDF5 format files, written with * and without using netCDF-4 parallel I/O. */ int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P}; /** Names for the output files. Two of them (pnetcdf and * classic) will be in classic netCDF format, the others * (serial4 and parallel4) will be in netCDF-4/HDF5 * format. All four can be read by the netCDF library, and all * will contain the same contents. */ char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"example2_pnetcdf.nc", "example2_classic.nc", "example2_serial4.nc", "example2_parallel4.nc"}; /** Number of processors that will do IO. In this example we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * example. */ int ioproc_stride = 1; /** Number of the aggregator? Always 0 in this example. */ int numAggregator = 0; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** Specifies the flavor of netCDF output format. */ int iotype; /** The dimension IDs. */ int dimids[NDIM]; /** Array index per processing unit. This is the number of * elements of the data array that will be handled by each * processor. In this example there are 16 data elements. If the * example is run on 4 processors, then arrIdxPerPe will be 4. */ PIO_Offset elements_per_pe; /** The ID for the parallel I/O system. It is set by * PIOc_Init_Intracomm(). It references an internal structure * containing the general IO subsystem data and MPI * structure. It is passed to PIOc_finalize() to free * associated resources, after all I/O, but before * MPI_Finalize is called. */ int iosysid; /** The ncid of the netCDF file created in this example. */ int ncid = 0; /** The ID of the netCDF varable in the example file. */ int varid; /** The I/O description ID as passed back by PIOc_InitDecomp() * and freed in PIOc_freedecomp(). */ int ioid; /** A buffer for sample data. The size of this array will * vary depending on how many processors are involved in the * execution of the example code. It's length will be the same * as elements_per_pe.*/ float *buffer; /** A buffer for reading data back from the file. The size of * this array will vary depending on how many processors are * involved in the execution of the example code. It's length * will be the same as elements_per_pe.*/ int *read_buffer; /** A 1-D array which holds the decomposition mapping for this * example. The size of this array will vary depending on how * many processors are involved in the execution of the * example code. It's length will be the same as * elements_per_pe. */ PIO_Offset *compdof; #ifdef HAVE_MPE /** MPE event numbers used to track start and stop of * different parts of the program for later display with * Jumpshot. */ int event_num[2][NUM_EVENTS]; #endif /* HAVE_MPE */ /** Needed for command line processing. */ int c; /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) { case 'v': verbose++; break; default: break; } #ifdef TIMING /* Initialize the GPTL timing library. */ int ret; if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); #ifdef HAVE_MPE /* Initialize MPE logging. */ if ((ret = MPE_Init_log())) ERR(ret); if (init_logging(my_rank, event_num)) ERR(ERR_LOGGING); /* Log with MPE that we are starting INIT. */ if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) { compdof[i] = my_rank * elements_per_pe + i + 1; } /* Create the PIO decomposition for this example. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); #ifdef HAVE_MPE /* Log with MPE that we are done with INIT. */ if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CREATE. */ if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename[fmt], format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], PIO_CLOBBER))) ERR(ret); /* Define netCDF dimensions and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); for (int d = 0; d < NDIM; d++) { if (verbose) printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, dim_name[d], dim_len[d]); if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); } if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); /* For netCDF-4 files, set the chunksize to improve performance. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CREATE. */ if ((ret = MPE_Log_event(event_num[END][CREATE_PNETCDF + fmt], 0, "end create"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Allocate space for sample data. */ if (!(buffer = malloc(elements_per_pe * sizeof(float)))) return PIO_ENOMEM; /* Write data for each timestep. */ for (int ts = 0; ts < NUM_TIMESTEPS; ts++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CALCULATE. */ if ((ret = MPE_Log_event(event_num[START][CALCULATE], 0, "start calculate"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Calculate sample data. Add some math function calls to make this slower. */ for (int i = 0; i < elements_per_pe; i++) if ((ret = calculate_value(my_rank, ts, &buffer[i]))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CALCULATE. */ if ((ret = MPE_Log_event(event_num[END][CALCULATE], 0, "end calculate"))) MPIERR(ret); /* Log with MPE that we are starting WRITE. */ if ((ret = MPE_Log_event(event_num[START][WRITE], 0, "start write"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Write data to the file. */ if (verbose) printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_setframe(ncid, varid, ts))) ERR(ret); if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe, buffer, NULL))) ERR(ret); if ((ret = PIOc_sync(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with WRITE. */ if ((ret = MPE_Log_event(event_num[END][WRITE], 0, "end write"))) MPIERR(ret); #endif /* HAVE_MPE */ } #ifdef HAVE_MPE /* Log with MPE that we are starting CLOSE. */ if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start close"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Free buffer space used in this example. */ free(buffer); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CLOSE. */ if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end close"))) MPIERR(ret); #endif /* HAVE_MPE */ /* After each file is closed, make all processors wait so that * all start creating the next file at the same time. */ if ((ret = MPI_Barrier(MPI_COMM_WORLD))) MPIERR(ret); } #ifdef HAVE_MPE /* Log with MPE that we are starting FREE. */ if ((ret = MPE_Log_event(event_num[START][FREE], 0, "start free"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with FREE. */ if ((ret = MPE_Log_event(event_num[END][FREE], 0, "end free"))) MPIERR(ret); /* Log with MPE that we are starting READ. */ if ((ret = MPE_Log_event(event_num[START][READ], 0, "start read"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Check the output file. */ /* if (!my_rank) */ /* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ /* if ((ret = check_file(ntasks, filename[fmt]))) */ /* ERR(ret); */ #ifdef HAVE_MPE /* Log with MPE that we are done with READ. */ if ((ret = MPE_Log_event(event_num[END][READ], 0, "end read"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif if (verbose) printf("rank: %d SUCCESS!\n", my_rank); return 0; }
/** @brief Main execution of code. Executes the functions to: - create a new examplePioClass instance - initialize MPI and the ParallelIO libraries - create the decomposition for this example - create the netCDF output file - define the variable in the file - write data to the variable in the file using decomposition - read the data back from the file using decomposition - close the file - clean up resources The example can be run from the command line (on system that support it) like this: <pre> mpiexec -n 4 ./examplePio </pre> The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump) for a 4-processor run: <pre> netcdf examplePio_c { dimensions: x = 16 ; variables: int foo(x) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ; } </pre> @param [in] argc argument count (should be zero) @param [in] argv argument array (should be NULL) @retval examplePioClass* Pointer to self. */ int main(int argc, char* argv[]) { /** Set to non-zero to get output to stdout. */ int verbose = 0; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Different output flavors. The example file is written (and * then read) four times. The first two flavors, * parallel-netcdf, and netCDF serial, both produce a netCDF * classic format file (but with different libraries). The * last two produce netCDF4/HDF5 format files, written with * and without using netCDF-4 parallel I/O. */ int format[NUM_NETCDF_FLAVORS]; /** Number of processors that will do IO. In this example we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * example. */ int ioproc_stride = 1; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** The dimension ID. */ int dimid; /** Array index per processing unit. This is the number of * elements of the data array that will be handled by each * processor. In this example there are 16 data elements. If the * example is run on 4 processors, then arrIdxPerPe will be 4. */ PIO_Offset elements_per_pe; /* Length of the dimensions in the data. This simple example * uses one-dimensional data. The lenght along that dimension * is DIM_LEN (16). */ int dim_len[1] = {DIM_LEN}; /** The ID for the parallel I/O system. It is set by * PIOc_Init_Intracomm(). It references an internal structure * containing the general IO subsystem data and MPI * structure. It is passed to PIOc_finalize() to free * associated resources, after all I/O, but before * MPI_Finalize is called. */ int iosysid; /** The ncid of the netCDF file created in this example. */ int ncid; /** The ID of the netCDF varable in the example file. */ int varid; /** The I/O description ID as passed back by PIOc_InitDecomp() * and freed in PIOc_freedecomp(). */ int ioid; /** A buffer for sample data. The size of this array will * vary depending on how many processors are involved in the * execution of the example code. It's length will be the same * as elements_per_pe.*/ int *buffer; /** A 1-D array which holds the decomposition mapping for this * example. The size of this array will vary depending on how * many processors are involved in the execution of the * example code. It's length will be the same as * elements_per_pe. */ PIO_Offset *compdof; /** Test filename. */ char filename[NC_MAX_NAME + 1]; /** The number of netCDF flavors available in this build. */ int num_flavors = 0; /** Used for command line processing. */ int c; /** Return value. */ int ret; /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) { case 'v': verbose++; break; default: break; } #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Turn on logging if available. */ /* PIOc_set_log_level(4); */ /* Change error handling to return errors. */ if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) return ret; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i + 1; /* Create the PIO decomposition for this example. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_INT, NDIM, dim_len, (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); /* The number of favors may change with the build parameters. */ #ifdef _PNETCDF format[num_flavors++] = PIO_IOTYPE_PNETCDF; #endif format[num_flavors++] = PIO_IOTYPE_NETCDF; #ifdef _NETCDF4 format[num_flavors++] = PIO_IOTYPE_NETCDF4C; format[num_flavors++] = PIO_IOTYPE_NETCDF4P; #endif /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* Create a filename. */ sprintf(filename, "example1_%d.nc", fmt); /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename, format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) ERR(ret); /* Define netCDF dimension and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); if ((ret = PIOc_def_dim(ncid, DIM_NAME, (PIO_Offset)dim_len[0], &dimid))) ERR(ret); if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, &dimid, &varid))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Prepare sample data. */ if (!(buffer = malloc(elements_per_pe * sizeof(int)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) buffer[i] = START_DATA_VAL + my_rank; /* Write data to the file. */ if (verbose) printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe, buffer, NULL))) ERR(ret); if ((ret = PIOc_sync(ncid))) ERR(ret); /* Free buffer space used in this example. */ free(buffer); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); } /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Check the output file. */ if (!my_rank) for (int fmt = 0; fmt < num_flavors; fmt++) { sprintf(filename, "example1_%d.nc", fmt); if ((ret = check_file(ntasks, filename))) ERR(ret); } /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif if (verbose) printf("rank: %d SUCCESS!\n", my_rank); return 0; }
/* Run tests for darray functions. */ int main(int argc, char **argv) { int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ MPI_Comm test_comm; /* A communicator for this test. */ int ret; /* Return code. */ /* Initialize test. */ if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, MIN_NTASKS, -1, &test_comm))) ERR(ERR_INIT); if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) return ret; /* Only do something on max_ntasks tasks. */ if (my_rank < TARGET_NTASKS) { int iosysid; /* The ID for the parallel I/O system. */ int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ int wioid, rioid; int maplen = MAPLEN; MPI_Offset wcompmap[MAPLEN]; MPI_Offset rcompmap[MAPLEN]; int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; /* Data we will write for each type. */ signed char byte_data[MAPLEN]; char char_data[MAPLEN]; short short_data[MAPLEN]; int int_data[MAPLEN]; float float_data[MAPLEN]; double double_data[MAPLEN]; #ifdef _NETCDF4 unsigned char ubyte_data[MAPLEN]; unsigned short ushort_data[MAPLEN]; unsigned int uint_data[MAPLEN]; long long int64_data[MAPLEN]; unsigned long long uint64_data[MAPLEN]; #endif /* _NETCDF4 */ /* Expected results for each type. */ signed char byte_expected[MAPLEN]; char char_expected[MAPLEN]; short short_expected[MAPLEN]; int int_expected[MAPLEN]; float float_expected[MAPLEN]; double double_expected[MAPLEN]; #ifdef _NETCDF4 unsigned char ubyte_expected[MAPLEN]; unsigned short ushort_expected[MAPLEN]; unsigned int uint_expected[MAPLEN]; long long int64_expected[MAPLEN]; unsigned long long uint64_expected[MAPLEN]; #endif /* _NETCDF4 */ /* Custom fill value for each type. */ signed char byte_fill = -2; char char_fill = 2; short short_fill = -2; int int_fill = -2; float float_fill = -2; double double_fill = -2; #ifdef _NETCDF4 unsigned char ubyte_fill = 2; unsigned short ushort_fill = 2; unsigned int uint_fill = 2; long long int64_fill = 2; unsigned long long uint64_fill = 2; #endif /* _NETCDF4 */ /* Default fill value for each type. */ signed char byte_default_fill = NC_FILL_BYTE; char char_default_fill = NC_FILL_CHAR; short short_default_fill = NC_FILL_SHORT; int int_default_fill = NC_FILL_INT; float float_default_fill = NC_FILL_FLOAT; double double_default_fill = NC_FILL_DOUBLE; #ifdef _NETCDF4 unsigned char ubyte_default_fill = NC_FILL_UBYTE; unsigned short ushort_default_fill = NC_FILL_USHORT; unsigned int uint_default_fill = NC_FILL_UINT; long long int64_default_fill = NC_FILL_INT64; unsigned long long uint64_default_fill = NC_FILL_UINT64; #endif /* _NETCDF4 */ int ret; /* Return code. */ /* Set up the compmaps. Don't forget these are 1-based * numbers, like in Fortran! */ for (int i = 0; i < MAPLEN; i++) { wcompmap[i] = i % 2 ? my_rank * MAPLEN + i + 1 : 0; /* Even values missing. */ rcompmap[i] = my_rank * MAPLEN + i + 1; } /* Figure out iotypes. */ if ((ret = get_iotypes(&num_flavors, flavor))) ERR(ret); /* Test for each rearranger. */ for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) { /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO_PROCS, ioproc_stride, ioproc_start, rearranger[r], &iosysid))) return ret; /* Test with and without custom fill values. */ for (int fv = 0; fv < NUM_TEST_CASES_FILLVALUE; fv++) { #ifndef _NETCDF4 #define NUM_TYPES 6 int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; #else #define NUM_TYPES 11 int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; #endif /* _NETCDF4 */ /* Determine what data to write. Put value of 42 into * array elements that will not get written. Due to * the decomposition, these will be replaced by fill * values. */ for (int i = 0; i < MAPLEN; i++) { byte_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; char_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; short_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; int_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; float_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; double_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; #ifdef _NETCDF4 ubyte_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; ushort_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; uint_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; int64_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; uint64_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42; #endif /* _NETCDF4 */ } /* Determine what data to expect from the test. For * even values of i, the fill value will be used, and * it may be custom or default fill value. */ for (int i = 0; i < MAPLEN; i++) { byte_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? byte_default_fill : byte_fill); char_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? char_default_fill : char_fill); short_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? short_default_fill : short_fill); int_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? int_default_fill : int_fill); float_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? float_default_fill : float_fill); double_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? double_default_fill : double_fill); #ifdef _NETCDF4 ubyte_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? ubyte_default_fill : ubyte_fill); ushort_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? ushort_default_fill : ushort_fill); uint_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? uint_default_fill : uint_fill); int64_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? int64_default_fill : int64_fill); uint64_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? uint64_default_fill : uint64_fill); #endif /* _NETCDF4 */ } /* Test for each available type. */ for (int t = 0; t < NUM_TYPES; t++) { void *expected; void *fill; void *data; int ncid, dimid, varid; char filename[NC_MAX_NAME + 1]; switch (test_type[t]) { case PIO_BYTE: expected = byte_expected; fill = fv ? &byte_default_fill : &byte_fill; data = byte_data; break; case PIO_CHAR: expected = char_expected; fill = fv ? &char_default_fill : &char_fill; data = char_data; break; case PIO_SHORT: expected = short_expected; fill = fv ? &short_default_fill : &short_fill; data = short_data; break; case PIO_INT: expected = int_expected; fill = fv ? &int_default_fill : &int_fill; data = int_data; break; case PIO_FLOAT: expected = float_expected; fill = fv ? &float_default_fill : &float_fill; data = float_data; break; case PIO_DOUBLE: expected = double_expected; fill = fv ? &double_default_fill : &double_fill; data = double_data; break; #ifdef _NETCDF4 case PIO_UBYTE: expected = ubyte_expected; fill = fv ? &ubyte_default_fill : &ubyte_fill; data = ubyte_data; break; case PIO_USHORT: expected = ushort_expected; fill = fv ? &ushort_default_fill : &ushort_fill; data = ushort_data; break; case PIO_UINT: expected = uint_expected; fill = fv ? &uint_default_fill : &uint_fill; data = uint_data; break; case PIO_INT64: expected = int64_expected; fill = fv ? &int64_default_fill : &int64_fill; data = int64_data; break; case PIO_UINT64: expected = uint64_expected; fill = fv ? &uint64_default_fill : &uint64_fill; data = uint64_data; break; #endif /* _NETCDF4 */ default: return ERR_AWFUL; } /* Initialize decompositions. */ if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, wcompmap, &wioid, &rearranger[r], NULL, NULL))) return ret; if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, rcompmap, &rioid, &rearranger[r], NULL, NULL))) return ret; /* Create the test file in each of the available iotypes. */ for (int fmt = 0; fmt < num_flavors; fmt++) { PIO_Offset type_size; void *data_in; /* Byte type doesn't work with pnetcdf. */ if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (test_type[t] == PIO_BYTE || test_type[t] == PIO_CHAR)) continue; /* NetCDF-4 types only work with netCDF-4 formats. */ if (test_type[t] > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P) continue; /* Put together filename. */ sprintf(filename, "%s_iotype_%d_rearr_%d_type_%d.nc", TEST_NAME, flavor[fmt], rearranger[r], test_type[t]); /* Create file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER))) return ret; /* Define metadata. */ if ((ret = PIOc_def_dim(ncid, DIM_NAME, dim_len[0], &dimid))) return ret; if ((ret = PIOc_def_var(ncid, VAR_NAME, test_type[t], NDIM1, &dimid, &varid))) return ret; if ((ret = PIOc_put_att(ncid, varid, FILL_VALUE_NAME, test_type[t], 1, fill))) return ret; if ((ret = PIOc_enddef(ncid))) return ret; /* Write some data. */ if ((ret = PIOc_write_darray(ncid, varid, wioid, MAPLEN, data, fill))) return ret; if ((ret = PIOc_sync(ncid))) return ret; /* What is size of type? */ if ((ret = PIOc_inq_type(ncid, test_type[t], NULL, &type_size))) return ret; /* Allocate space to read data into. */ if (!(data_in = malloc(type_size * MAPLEN))) return PIO_ENOMEM; /* Read the data. */ if ((ret = PIOc_read_darray(ncid, varid, rioid, MAPLEN, data_in))) return ret; /* Check results. */ if (memcmp(data_in, expected, type_size * MAPLEN)) return ERR_AWFUL; /* Release storage. */ free(data_in); /* Close file. */ if ((ret = PIOc_closefile(ncid))) return ret; } /* next iotype */ /* Free decompositions. */ if ((ret = PIOc_freedecomp(iosysid, wioid))) return ret; if ((ret = PIOc_freedecomp(iosysid, rioid))) return ret; } /* next type */ } /* next fill value test case */ } /* next rearranger */ /* Finalize PIO system. */ if ((ret = PIOc_finalize(iosysid))) return ret; } /* endif my_rank < TARGET_NTASKS */ /* Finalize the MPI library. */ if ((ret = pio_test_finalize(&test_comm))) return ret; printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); return 0; }
/** ** @ingroup PIO_closefile ** @brief close a file previously opened with PIO ** @param ncid: the file pointer */ int PIOc_closefile(int ncid) { int ierr; int msg; int mpierr; iosystem_desc_t *ios; file_desc_t *file; ierr = PIO_NOERR; file = pio_get_file_from_id(ncid); if(file == NULL) return PIO_EBADID; ios = file->iosystem; msg = 0; if((file->mode & PIO_WRITE)){ PIOc_sync(ncid); } if(ios->async_interface && ! ios->ioproc){ if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_close(file->fh); break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ ierr = nc_close(file->fh); } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: if((file->mode & PIO_WRITE)){ ierr = ncmpi_buffer_detach(file->fh); } ierr = ncmpi_close(file->fh); break; #endif default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } } if(ios->io_rank==0){ printf("Close file %d \n",file->fh); // if(file->fh==5) print_trace(stdout); } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); int iret = pio_delete_file_from_list(ncid); return ierr; }
/* Write, then read, a simple example with darrays. The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump): <pre> netcdf darray_no_async_iotype_1 { dimensions: unlimted = UNLIMITED ; // (2 currently) x = 4 ; y = 4 ; variables: int foo(unlimted, x, y) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45, 142, 142, 142, 142, 143, 143, 143, 143, 144, 144, 144, 144, 145, 145, 145, 145 ; } </pre> */ int main(int argc, char* argv[]) { int my_rank; /* Zero-based rank of processor. */ int ntasks; /* Number of processors involved in current execution. */ int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ int ioproc_start = 0; /* Rank of first task to be used for I/O. */ PIO_Offset elements_per_pe; /* Array elements per processing unit. */ int iosysid; /* The ID for the parallel I/O system. */ int ncid; /* The ncid of the netCDF file. */ int dimid[NDIM3]; /* The dimension ID. */ int varid; /* The ID of the netCDF varable. */ int ioid; /* The I/O description ID. */ char filename[NC_MAX_NAME + 1]; /* Test filename. */ int num_flavors = 0; /* Number of iotypes available in this build. */ int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */ int ret; /* Return value. */ #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (ntasks != TARGET_NTASKS) fprintf(stderr, "Number of processors must be 4!\n"); printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n", my_rank, ntasks); /* Turn on logging. */ if ((ret = PIOc_set_log_level(LOG_LEVEL))) return ret; /* Initialize the PIO IO system. This specifies how many and * which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride, ioproc_start, PIO_REARR_BOX, &iosysid))) ERR(ret); /* Describe the decomposition. */ elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS; /* Allocate and initialize array of decomposition mapping. */ PIO_Offset compdof[elements_per_pe]; for (int i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i; /* Create the PIO decomposition for this example. Since this * is a variable with an unlimited dimension, we want to * create a 2-D composition which represents one record. */ printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, compdof, &ioid, 0, NULL, NULL))) ERR(ret); /* The number of favors may change with the build parameters. */ #ifdef _PNETCDF format[num_flavors++] = PIO_IOTYPE_PNETCDF; #endif format[num_flavors++] = PIO_IOTYPE_NETCDF; #ifdef _NETCDF4 format[num_flavors++] = PIO_IOTYPE_NETCDF4C; format[num_flavors++] = PIO_IOTYPE_NETCDF4P; #endif /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* Create a filename. */ sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); /* Create the netCDF output file. */ printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename, format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) ERR(ret); /* Define netCDF dimension and variable. */ printf("rank: %d Defining netCDF metadata...\n", my_rank); for (int d = 0; d < NDIM3; d++) if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) ERR(ret); if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Allocate storage for sample data. */ int buffer[elements_per_pe]; /* Write each timestep. */ for (int t = 0; t < NUM_TIMESTEPS; t++) { /* Create some data for this timestep. */ for (int i = 0; i < elements_per_pe; i++) buffer[i] = 100 * t + START_DATA_VAL + my_rank; /* Write data to the file. */ printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_setframe(ncid, varid, t))) ERR(ret); if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) ERR(ret); } /* THis will cause all data to be written to disk. */ if ((ret = PIOc_sync(ncid))) ERR(ret); /* Close the netCDF file. */ printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); /* Check the output file. */ /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */ /* my_rank, ioid))) */ /* ERR(ret); */ } /* Free the PIO decomposition. */ printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif printf("rank: %d SUCCESS!\n", my_rank); return 0; }