/** Check the dimension names. * * @param my_rank rank of process * @param ncid ncid of open netCDF file * * @returns 0 for success, error code otherwise. */ int check_dim_names(int my_rank, int ncid, int verbose) { char dim_name[NC_MAX_NAME + 1]; char zero_dim_name[NC_MAX_NAME + 1]; int ret; for (int d = 0; d < NDIM; d++) { strcpy(dim_name, "11111111111111111111111111111111"); if ((ret = PIOc_inq_dimname(ncid, d, dim_name))) return ret; if (verbose) printf("my_rank %d dim %d name %s\n", my_rank, d, dim_name); /* Did other ranks get the same name? */ if (!my_rank) strcpy(zero_dim_name, dim_name); /* if (verbose) */ /* printf("rank %d dim_name %s zero_dim_name %s\n", my_rank, dim_name, zero_dim_name); */ if ((ret = MPI_Bcast(&zero_dim_name, strlen(dim_name) + 1, MPI_CHAR, 0, MPI_COMM_WORLD))) MPIERR(ret); if (strcmp(dim_name, zero_dim_name)) return ERR_AWFUL; } return 0; }
/** * Run all the tests. * * @param iosysid the IO system ID. * @param num_flavors number of available iotypes in the build. * @param flavor pointer to array of the available iotypes. * @param my_rank rank of this task. * @param rearranger the rearranger to use (PIO_REARR_BOX or * PIO_REARR_SUBSET). * @param test_comm the communicator the test is running on. * @returns 0 for success, error code otherwise. */ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, int rearranger, MPI_Comm test_comm) { int ioid; int my_test_size; int ret; /* Return code. */ if ((ret = MPI_Comm_size(test_comm, &my_test_size))) MPIERR(ret); /* Decompose the data over the tasks. */ if ((ret = create_decomposition_3d(TARGET_NTASKS, my_rank, iosysid, &ioid))) return ret; /* Test decomposition read/write. */ if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, rearranger, test_comm))) return ret; /* Test with/without providing a fill value to PIOc_write_darray(). */ for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) { /* Run a simple darray test. */ if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, provide_fill))) return ret; } /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); return PIO_NOERR; }
/** Check the attribute name. * * @param my_rank rank of process * @param ncid ncid of open netCDF file * * @returns 0 for success, error code otherwise. */ int check_att_name(int my_rank, int ncid, int verbose) { char att_name[NC_MAX_NAME + 1]; char zero_att_name[NC_MAX_NAME + 1]; int ret; strcpy(att_name, "11111111111111111111111111111111"); if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, att_name))) return ret; if (verbose) printf("my_rank %d att name %s\n", my_rank, att_name); /* Did everyone ranks get the same length name? */ /* if (strlen(att_name) != strlen(ATT_NAME)) return ERR_AWFUL;*/ if (!my_rank) strcpy(zero_att_name, att_name); if ((ret = MPI_Bcast(&zero_att_name, strlen(att_name) + 1, MPI_CHAR, 0, MPI_COMM_WORLD))) MPIERR(ret); if (strcmp(att_name, zero_att_name)) return ERR_AWFUL; return 0; }
/** Check the variable name. * * @param my_rank rank of process * @param ncid ncid of open netCDF file * * @returns 0 for success, error code otherwise. */ int check_var_name(int my_rank, int ncid, int verbose) { char var_name[NC_MAX_NAME + 1]; char zero_var_name[NC_MAX_NAME + 1]; int ret; strcpy(var_name, "11111111111111111111111111111111"); if ((ret = PIOc_inq_varname(ncid, 0, var_name))) return ret; if (verbose) printf("my_rank %d var name %s\n", my_rank, var_name); /* Did other ranks get the same name? */ if (!my_rank) strcpy(zero_var_name, var_name); if ((ret = MPI_Bcast(&zero_var_name, strlen(var_name) + 1, MPI_CHAR, 0, MPI_COMM_WORLD))) MPIERR(ret); if (strcmp(var_name, zero_var_name)) return ERR_AWFUL; return 0; }
int main(int argc, char* argv[]) { int n, my_rank; int array_of_subsizes[NDIMS], array_of_starts[NDIMS], array_of_sizes[NDIMS]; int size = 4; int sqrtn; int ln; MPI_Datatype filetype, memtype; MPI_File fh; char hdr[128]; int header_bytes; unsigned char *cur; char name[128]; int resultlen; int ret; int i, j; /* Initialize MPI. */ MPI_Init(&argc, &argv); MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN); /* Learn my rank and the total number of processors. */ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); MPI_Comm_size(MPI_COMM_WORLD, &n); /* Speak! */ MPI_Get_processor_name(name, &resultlen); printf("process %d running on %s\n", my_rank, name); /* Set up our values. */ sqrtn = (int)sqrt(n); ln = size/sqrtn; printf("n = %d, sqrtn = %d, ln = %d storage = %d\n", n, sqrtn, ln, (ln + 2) * (ln + 2)); /* Allocation storage. */ if (!(cur = calloc((ln + 2) * (ln + 2), 1))) return ERR; /* Initialize data. */ for (i = 1; i < ln + 1; i++) for (j = 1; j < ln + 1; j++) cur[i * (ln + 2) + j] = my_rank; /* Create a subarray type for the file. */ array_of_sizes[0] = array_of_sizes[1] = size; array_of_subsizes[0] = array_of_subsizes[1] = ln; array_of_starts[0] = my_rank/sqrtn * ln; array_of_starts[1] = (my_rank % sqrtn) * ln; if ((ret = MPI_Type_create_subarray(NDIMS, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_BYTE, &filetype))) MPIERR(ret); if ((ret = MPI_Type_commit(&filetype))) MPIERR(ret); /* Create a subarray type for memory. */ array_of_sizes[0] = array_of_sizes[1] = ln + 2; array_of_subsizes[0] = array_of_subsizes[1] = ln; array_of_starts[0] = array_of_starts[1] = 1; if ((ret = MPI_Type_create_subarray(NDIMS, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_BYTE, &memtype))) MPIERR(ret); if ((ret = MPI_Type_commit(&memtype))) MPIERR(ret); MPI_File_delete(FILE_NAME, MPI_INFO_NULL); if ((ret = MPI_File_open(MPI_COMM_WORLD, FILE_NAME, MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &fh))) MPIERR(ret); /* Create header info, and have process 0 write it to the file. */ sprintf(hdr, "P5\n%d %d\n255\n", size, size); header_bytes = strlen(hdr); if ((ret = MPI_File_write_all(fh, hdr, header_bytes, MPI_BYTE, MPI_STATUS_IGNORE))) MPIERR(ret); /* Set the file view to translate our memory data into the file's data layout. */ MPI_File_set_view(fh, header_bytes, MPI_BYTE, filetype, "native", MPI_INFO_NULL); /* Write the output. */ MPI_File_write(fh, cur, 1, memtype, MPI_STATUS_IGNORE); if ((ret = MPI_File_close(&fh))) MPIERR(ret); MPI_Finalize(); return 0; }
/** Run Tests for NetCDF-4 Functions. * * @param argc argument count * @param argv array of arguments */ int main(int argc, char **argv) { int verbose = 1; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Different output flavors. The example file is written (and * then read) four times. The first two flavors, * parallel-netcdf, and netCDF serial, both produce a netCDF * classic format file (but with different libraries). The * last two produce netCDF4/HDF5 format files, written with * and without using netCDF-4 parallel I/O. */ int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P}; /** Names for the output files. Two of them (pnetcdf and * classic) will be in classic netCDF format, the others * (serial4 and parallel4) will be in netCDF-4/HDF5 * format. All four can be read by the netCDF library, and all * will contain the same contents. */ char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc", "test_nc4_classic.nc", "test_nc4_serial4.nc", "test_nc4_parallel4.nc"}; /** Number of processors that will do IO. In this example we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * example. */ int ioproc_stride = 1; /** Number of the aggregator? Always 0 in this example. */ int numAggregator = 0; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** Specifies the flavor of netCDF output format. */ int iotype; /** The dimension IDs. */ int dimids[NDIM]; /** Array index per processing unit. This is the number of * elements of the data array that will be handled by each * processor. In this example there are 16 data elements. If the * example is run on 4 processors, then arrIdxPerPe will be 4. */ PIO_Offset elements_per_pe; /** The ID for the parallel I/O system. It is set by * PIOc_Init_Intracomm(). It references an internal structure * containing the general IO subsystem data and MPI * structure. It is passed to PIOc_finalize() to free * associated resources, after all I/O, but before * MPI_Finalize is called. */ int iosysid; /** The ncid of the netCDF file created in this example. */ int ncid = 0; /** The ID of the netCDF varable in the example file. */ int varid; /** The I/O description ID as passed back by PIOc_InitDecomp() * and freed in PIOc_freedecomp(). */ int ioid; /** A buffer for sample data. The size of this array will * vary depending on how many processors are involved in the * execution of the example code. It's length will be the same * as elements_per_pe.*/ float *buffer; /** A buffer for reading data back from the file. The size of * this array will vary depending on how many processors are * involved in the execution of the example code. It's length * will be the same as elements_per_pe.*/ int *read_buffer; /** A 1-D array which holds the decomposition mapping for this * example. The size of this array will vary depending on how * many processors are involved in the execution of the * example code. It's length will be the same as * elements_per_pe. */ PIO_Offset *compdof; /** Return code. */ int ret; #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) { compdof[i] = my_rank * elements_per_pe + i + 1; } /* Create the PIO decomposition for this test. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); #ifdef HAVE_MPE /* Log with MPE that we are done with INIT. */ if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CREATE. */ if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename[fmt], format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], PIO_CLOBBER))) ERR(ret); /* Define netCDF dimensions and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); for (int d = 0; d < NDIM; d++) { if (verbose) printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, dim_name[d], dim_len[d]); if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); } if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); /* For netCDF-4 files, set the chunksize to improve performance. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) { if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize))) ERR(ret); /** Check that the inq_var_chunking function works. */ int storage; size_t my_chunksize[NDIM]; if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize))) ERR(ret); /** For serial netCDF-4, only processor rank 0 gets the answers. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank || format[fmt] == PIO_IOTYPE_NETCDF4P) { if (storage != NC_CHUNKED) ERR(ERR_AWFUL); for (int d = 0; d < NDIM; d++) if (my_chunksize[d] != chunksize[d]) ERR(ERR_AWFUL); } /* Check that the inv_var_deflate functions works. */ int shuffle; int deflate; int deflate_level; if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level))) ERR(ret); /** For serial netCDF-4, only processor rank 0 gets the * answers. Also deflate is turned on by default */ if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank) if (shuffle || !deflate || deflate_level != 1) ERR(ERR_AWFUL); /* For parallel netCDF, no compression available. :-( */ if (format[fmt] == PIO_IOTYPE_NETCDF4P) if (shuffle || deflate) ERR(ERR_AWFUL); } else { /* Trying to set chunking for non-netCDF-4 files results * in the PIO_ENOTNC4 error. */ if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)) != PIO_ENOTNC4) ERR(ERR_AWFUL); } if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); } /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif return 0; }
/** Run Tests for NetCDF-4 Functions. * * @param argc argument count * @param argv array of arguments */ int main(int argc, char **argv) { int verbose = 1; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Specifies the flavor of netCDF output format. */ int iotype; /** Different output flavors. */ int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P}; /** Names for the output files. */ char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_names_pnetcdf.nc", "test_names_classic.nc", "test_names_serial4.nc", "test_names_parallel4.nc"}; /** Number of processors that will do IO. In this test we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * test. */ int ioproc_stride = 1; /** Number of the aggregator? Always 0 in this test. */ int numAggregator = 0; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** The dimension IDs. */ int dimids[NDIM]; /** Array index per processing unit. */ PIO_Offset elements_per_pe; /** The ID for the parallel I/O system. */ int iosysid; /** The ncid of the netCDF file. */ int ncid = 0; /** The ID of the netCDF varable. */ int varid; /** Storage of netCDF-4 files (contiguous vs. chunked). */ int storage; /** Chunksizes set in the file. */ size_t my_chunksize[NDIM]; /** The shuffle filter setting in the netCDF-4 test file. */ int shuffle; /** Non-zero if deflate set for the variable in the netCDF-4 test file. */ int deflate; /** The deflate level set for the variable in the netCDF-4 test file. */ int deflate_level; /** Non-zero if fletcher32 filter is used for variable. */ int fletcher32; /** Endianness of variable. */ int endianness; /* Size of the file chunk cache. */ size_t chunk_cache_size; /* Number of elements in file cache. */ size_t nelems; /* File cache preemption. */ float preemption; /* Size of the var chunk cache. */ size_t var_cache_size; /* Number of elements in var cache. */ size_t var_cache_nelems; /* Var cache preemption. */ float var_cache_preemption; /** The I/O description ID. */ int ioid; /** A buffer for sample data. */ float *buffer; /** A buffer for reading data back from the file. */ int *read_buffer; /** The decomposition mapping. */ PIO_Offset *compdof; /** Return code. */ int ret; /** Index for loops. */ int fmt, d, d1, i; #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (i = 0; i < elements_per_pe; i++) { compdof[i] = my_rank * elements_per_pe + i + 1; } /* Create the PIO decomposition for this test. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); /* How many flavors will we be running for? */ int num_flavors = 0; int fmtidx = 0; #ifdef _PNETCDF num_flavors++; format[fmtidx++] = PIO_IOTYPE_PNETCDF; #endif #ifdef _NETCDF num_flavors++; format[fmtidx++] = PIO_IOTYPE_NETCDF; #endif #ifdef _NETCDF4 num_flavors += 2; format[fmtidx++] = PIO_IOTYPE_NETCDF4C; format[fmtidx] = PIO_IOTYPE_NETCDF4P; #endif /* Use PIO to create the example file in each of the four * available ways. */ for (fmt = 0; fmt < num_flavors; fmt++) { /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename[fmt], format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], PIO_CLOBBER))) ERR(ret); /* Define netCDF dimensions and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); for (d = 0; d < NDIM; d++) { if (verbose) printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, dim_name[d], dim_len[d]); if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); } /* Check the dimension names. */ if ((ret = check_dim_names(my_rank, ncid, verbose))) ERR(ret); /* Define a global attribute. */ int att_val = 42; if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_val))) ERR(ret); /* Check the attribute name. */ if ((ret = check_att_name(my_rank, ncid, verbose))) ERR(ret); /* Define a variable. */ if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); /* Check the variable name. */ if ((ret = check_var_name(my_rank, ncid, verbose))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); /* Put a barrier here to make verbose output look better. */ if ((ret = MPI_Barrier(MPI_COMM_WORLD))) MPIERR(ret); } /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif return 0; }
/** @brief Main execution of code. Executes the functions to: - create a new examplePioClass instance - initialize MPI and the ParallelIO libraries - create the decomposition for this example - create the netCDF output file - define the variable in the file - write data to the variable in the file using decomposition - read the data back from the file using decomposition - close the file - clean up resources The example can be run from the command line (on system that support it) like this: <pre> mpiexec -n 4 ./examplePio </pre> The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump) for a 4-processor run: <pre> netcdf examplePio_c { dimensions: x = 16 ; variables: int foo(x) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ; } </pre> @param [in] argc argument count (should be zero) @param [in] argv argument array (should be NULL) @retval examplePioClass* Pointer to self. */ int main(int argc, char* argv[]) { /** Set to non-zero to get output to stdout. */ int verbose = 0; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Different output flavors. The example file is written (and * then read) four times. The first two flavors, * parallel-netcdf, and netCDF serial, both produce a netCDF * classic format file (but with different libraries). The * last two produce netCDF4/HDF5 format files, written with * and without using netCDF-4 parallel I/O. */ int format[NUM_NETCDF_FLAVORS]; /** Number of processors that will do IO. In this example we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * example. */ int ioproc_stride = 1; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** The dimension ID. */ int dimid; /** Array index per processing unit. This is the number of * elements of the data array that will be handled by each * processor. In this example there are 16 data elements. If the * example is run on 4 processors, then arrIdxPerPe will be 4. */ PIO_Offset elements_per_pe; /* Length of the dimensions in the data. This simple example * uses one-dimensional data. The lenght along that dimension * is DIM_LEN (16). */ int dim_len[1] = {DIM_LEN}; /** The ID for the parallel I/O system. It is set by * PIOc_Init_Intracomm(). It references an internal structure * containing the general IO subsystem data and MPI * structure. It is passed to PIOc_finalize() to free * associated resources, after all I/O, but before * MPI_Finalize is called. */ int iosysid; /** The ncid of the netCDF file created in this example. */ int ncid; /** The ID of the netCDF varable in the example file. */ int varid; /** The I/O description ID as passed back by PIOc_InitDecomp() * and freed in PIOc_freedecomp(). */ int ioid; /** A buffer for sample data. The size of this array will * vary depending on how many processors are involved in the * execution of the example code. It's length will be the same * as elements_per_pe.*/ int *buffer; /** A 1-D array which holds the decomposition mapping for this * example. The size of this array will vary depending on how * many processors are involved in the execution of the * example code. It's length will be the same as * elements_per_pe. */ PIO_Offset *compdof; /** Test filename. */ char filename[NC_MAX_NAME + 1]; /** The number of netCDF flavors available in this build. */ int num_flavors = 0; /** Used for command line processing. */ int c; /** Return value. */ int ret; /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) { case 'v': verbose++; break; default: break; } #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Turn on logging if available. */ /* PIOc_set_log_level(4); */ /* Change error handling to return errors. */ if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) return ret; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i + 1; /* Create the PIO decomposition for this example. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_INT, NDIM, dim_len, (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); /* The number of favors may change with the build parameters. */ #ifdef _PNETCDF format[num_flavors++] = PIO_IOTYPE_PNETCDF; #endif format[num_flavors++] = PIO_IOTYPE_NETCDF; #ifdef _NETCDF4 format[num_flavors++] = PIO_IOTYPE_NETCDF4C; format[num_flavors++] = PIO_IOTYPE_NETCDF4P; #endif /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* Create a filename. */ sprintf(filename, "example1_%d.nc", fmt); /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename, format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) ERR(ret); /* Define netCDF dimension and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); if ((ret = PIOc_def_dim(ncid, DIM_NAME, (PIO_Offset)dim_len[0], &dimid))) ERR(ret); if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, &dimid, &varid))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Prepare sample data. */ if (!(buffer = malloc(elements_per_pe * sizeof(int)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) buffer[i] = START_DATA_VAL + my_rank; /* Write data to the file. */ if (verbose) printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe, buffer, NULL))) ERR(ret); if ((ret = PIOc_sync(ncid))) ERR(ret); /* Free buffer space used in this example. */ free(buffer); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); } /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Check the output file. */ if (!my_rank) for (int fmt = 0; fmt < num_flavors; fmt++) { sprintf(filename, "example1_%d.nc", fmt); if ((ret = check_file(ntasks, filename))) ERR(ret); } /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif if (verbose) printf("rank: %d SUCCESS!\n", my_rank); return 0; }
/** Main execution of code. Executes the functions to: - create a new examplePioClass instance - initialize MPI and the ParallelIO libraries - create the decomposition for this example - create the netCDF output file - define the variable in the file - write data to the variable in the file using decomposition - read the data back from the file using decomposition - close the file - clean up resources The example can be run from the command line (on system that support it) like this: <pre> mpiexec -n 4 ./examplePio </pre> The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump) for a 4-processor run: <pre> netcdf examplePio_c { dimensions: x = 16 ; variables: int foo(x) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ; } </pre> @param [in] argc argument count (should be zero) @param [in] argv argument array (should be NULL) @retval examplePioClass* Pointer to self. */ int main(int argc, char* argv[]) { /** Set to non-zero to get output to stdout. */ int verbose = 0; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Different output flavors. The example file is written (and * then read) four times. The first two flavors, * parallel-netcdf, and netCDF serial, both produce a netCDF * classic format file (but with different libraries). The * last two produce netCDF4/HDF5 format files, written with * and without using netCDF-4 parallel I/O. */ int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P}; /** Names for the output files. Two of them (pnetcdf and * classic) will be in classic netCDF format, the others * (serial4 and parallel4) will be in netCDF-4/HDF5 * format. All four can be read by the netCDF library, and all * will contain the same contents. */ char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"example2_pnetcdf.nc", "example2_classic.nc", "example2_serial4.nc", "example2_parallel4.nc"}; /** Number of processors that will do IO. In this example we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * example. */ int ioproc_stride = 1; /** Number of the aggregator? Always 0 in this example. */ int numAggregator = 0; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** Specifies the flavor of netCDF output format. */ int iotype; /** The dimension IDs. */ int dimids[NDIM]; /** Array index per processing unit. This is the number of * elements of the data array that will be handled by each * processor. In this example there are 16 data elements. If the * example is run on 4 processors, then arrIdxPerPe will be 4. */ PIO_Offset elements_per_pe; /** The ID for the parallel I/O system. It is set by * PIOc_Init_Intracomm(). It references an internal structure * containing the general IO subsystem data and MPI * structure. It is passed to PIOc_finalize() to free * associated resources, after all I/O, but before * MPI_Finalize is called. */ int iosysid; /** The ncid of the netCDF file created in this example. */ int ncid = 0; /** The ID of the netCDF varable in the example file. */ int varid; /** The I/O description ID as passed back by PIOc_InitDecomp() * and freed in PIOc_freedecomp(). */ int ioid; /** A buffer for sample data. The size of this array will * vary depending on how many processors are involved in the * execution of the example code. It's length will be the same * as elements_per_pe.*/ float *buffer; /** A buffer for reading data back from the file. The size of * this array will vary depending on how many processors are * involved in the execution of the example code. It's length * will be the same as elements_per_pe.*/ int *read_buffer; /** A 1-D array which holds the decomposition mapping for this * example. The size of this array will vary depending on how * many processors are involved in the execution of the * example code. It's length will be the same as * elements_per_pe. */ PIO_Offset *compdof; #ifdef HAVE_MPE /** MPE event numbers used to track start and stop of * different parts of the program for later display with * Jumpshot. */ int event_num[2][NUM_EVENTS]; #endif /* HAVE_MPE */ /** Needed for command line processing. */ int c; /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) { case 'v': verbose++; break; default: break; } #ifdef TIMING /* Initialize the GPTL timing library. */ int ret; if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); #ifdef HAVE_MPE /* Initialize MPE logging. */ if ((ret = MPE_Init_log())) ERR(ret); if (init_logging(my_rank, event_num)) ERR(ERR_LOGGING); /* Log with MPE that we are starting INIT. */ if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) { compdof[i] = my_rank * elements_per_pe + i + 1; } /* Create the PIO decomposition for this example. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); #ifdef HAVE_MPE /* Log with MPE that we are done with INIT. */ if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CREATE. */ if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename[fmt], format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], PIO_CLOBBER))) ERR(ret); /* Define netCDF dimensions and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); for (int d = 0; d < NDIM; d++) { if (verbose) printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, dim_name[d], dim_len[d]); if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); } if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); /* For netCDF-4 files, set the chunksize to improve performance. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CREATE. */ if ((ret = MPE_Log_event(event_num[END][CREATE_PNETCDF + fmt], 0, "end create"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Allocate space for sample data. */ if (!(buffer = malloc(elements_per_pe * sizeof(float)))) return PIO_ENOMEM; /* Write data for each timestep. */ for (int ts = 0; ts < NUM_TIMESTEPS; ts++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CALCULATE. */ if ((ret = MPE_Log_event(event_num[START][CALCULATE], 0, "start calculate"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Calculate sample data. Add some math function calls to make this slower. */ for (int i = 0; i < elements_per_pe; i++) if ((ret = calculate_value(my_rank, ts, &buffer[i]))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CALCULATE. */ if ((ret = MPE_Log_event(event_num[END][CALCULATE], 0, "end calculate"))) MPIERR(ret); /* Log with MPE that we are starting WRITE. */ if ((ret = MPE_Log_event(event_num[START][WRITE], 0, "start write"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Write data to the file. */ if (verbose) printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_setframe(ncid, varid, ts))) ERR(ret); if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe, buffer, NULL))) ERR(ret); if ((ret = PIOc_sync(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with WRITE. */ if ((ret = MPE_Log_event(event_num[END][WRITE], 0, "end write"))) MPIERR(ret); #endif /* HAVE_MPE */ } #ifdef HAVE_MPE /* Log with MPE that we are starting CLOSE. */ if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start close"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Free buffer space used in this example. */ free(buffer); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with CLOSE. */ if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end close"))) MPIERR(ret); #endif /* HAVE_MPE */ /* After each file is closed, make all processors wait so that * all start creating the next file at the same time. */ if ((ret = MPI_Barrier(MPI_COMM_WORLD))) MPIERR(ret); } #ifdef HAVE_MPE /* Log with MPE that we are starting FREE. */ if ((ret = MPE_Log_event(event_num[START][FREE], 0, "start free"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); #ifdef HAVE_MPE /* Log with MPE that we are done with FREE. */ if ((ret = MPE_Log_event(event_num[END][FREE], 0, "end free"))) MPIERR(ret); /* Log with MPE that we are starting READ. */ if ((ret = MPE_Log_event(event_num[START][READ], 0, "start read"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Check the output file. */ /* if (!my_rank) */ /* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ /* if ((ret = check_file(ntasks, filename[fmt]))) */ /* ERR(ret); */ #ifdef HAVE_MPE /* Log with MPE that we are done with READ. */ if ((ret = MPE_Log_event(event_num[END][READ], 0, "end read"))) MPIERR(ret); #endif /* HAVE_MPE */ /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif if (verbose) printf("rank: %d SUCCESS!\n", my_rank); return 0; }
/** @brief Main execution of code. Executes the functions to: - create a new examplePioClass instance - initialize MPI and the ParallelIO libraries - create the decomposition for this example - create the netCDF output file - define the variable in the file - write data to the variable in the file using decomposition - read the data back from the file using decomposition - close the file - clean up resources The example can be run from the command line (on system that support it) like this: <pre> mpiexec -n 4 ./examplePio </pre> The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump) for a 4-processor run: <pre> netcdf examplePio_c { dimensions: x = 16 ; variables: int foo(x) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ; } </pre> @param [in] argc argument count (should be zero) @param [in] argv argument array (should be NULL) @retval examplePioClass* Pointer to self. */ int main(int argc, char* argv[]) { /** Set to non-zero to get output to stdout. */ int verbose = 0; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Number of processors that will do IO. In this example we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * example. */ int stride = 1; /** Number of the aggregator? Always 0 in this example. */ int numAggregator = 0; /** */ int optBase = 1; /** Specifies the flavor of netCDF output format. */ int iotype; /** The dimension ID. */ int pioDimId; /** */ PIO_Offset ista; /** */ PIO_Offset isto; /** Array index per processing unit. This is the number of * elements of the data array that will be handled by each * processor. In this example there are 16 data elements. If the * example is run on 4 processors, then arrIdxPerPe will be 4. */ PIO_Offset arrIdxPerPe; /* Length of the dimension in data. */ int dimLen[1]; /** The ID for the parallel I/O system. It is set by * PIOc_Init_Intracomm(). It references an internal structure * containing the general IO subsystem data and MPI structure. */ int pio_io_system; /** The ncid of the netCDF file created in this example. */ int pioFileDesc; /** The ID of the netCDF varable in the example file. */ int pioVar; /** The I/O description ID as passed back by PIOc_InitDecomp(). */ int iodescNCells; /** A buffer for sample data. */ int *dataBuffer; /** A buffer for reading data back from the file. */ int *readBuffer; /** A 1-D array which holds the decomposition mapping for this * example. */ PIO_Offset *compdof; /** The example file name. */ char file_name[] = EXAMPLE_FILENAME; /** Used for command line processing. */ int c; /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) { case 'v': verbose++; break; default: break; } #ifdef TIMING /* Initialize the GPTL timing library. */ int ret; if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); /* Initialize the ParallelIO library IO system. */ iotype = PIO_IOTYPE_NETCDF; /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Initialize the IO system. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, stride, optBase, PIO_REARR_SUBSET, &pio_io_system))) ERR(ret); /* Finalize the IO system. */ if ((ret = PIOc_finalize(pio_io_system))) ERR(ret); /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif return 0; }
/* Run async tests. */ int main(int argc, char **argv) { int my_rank; /* Zero-based rank of processor. */ int ntasks; /* Number of processors involved in current execution. */ int iosysid_world; /* The ID for the parallel I/O system. */ int even_iosysid; /* The ID for iosystem of even_comm. */ int overlap_iosysid; /* The ID for iosystem of even_comm. */ MPI_Group world_group; /* An MPI group of world. */ MPI_Group even_group; /* An MPI group of 0 and 2. */ MPI_Group overlap_group; /* An MPI group of 0, 1, and 3. */ MPI_Comm even_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 2 */ MPI_Comm overlap_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 1, 2. */ int even_rank = -1, overlap_rank = -1; /* Tasks rank in communicator. */ int even_size = 0, overlap_size = 0; /* Size of communicator. */ int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ MPI_Comm test_comm; int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; int ret; /* Return code. */ /* Initialize test. */ if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, TARGET_NTASKS, -1, &test_comm))) ERR(ERR_INIT); /* Test code runs on TARGET_NTASKS tasks. The left over tasks do * nothing. */ if (my_rank < TARGET_NTASKS) { /* Figure out iotypes. */ if ((ret = get_iotypes(&num_flavors, flavor))) ERR(ret); /* Test with both rearrangers. */ for (int r = 0; r < NUM_REARRANGERS; r++) { /* Initialize PIO system on world. */ if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, rearranger[r], &iosysid_world))) ERR(ret); /* Set the error handler. */ if ((ret = PIOc_set_iosystem_error_handling(iosysid_world, PIO_BCAST_ERROR, NULL))) ERR(ret); /* Get MPI_Group of world comm. */ if ((ret = MPI_Comm_group(test_comm, &world_group))) ERR(ret); /* Create a group with tasks 0 and 2. */ int even_ranges[EVEN_NUM_RANGES][3] = {{0, 2, 2}}; if ((ret = MPI_Group_range_incl(world_group, EVEN_NUM_RANGES, even_ranges, &even_group))) ERR(ret); /* Create a communicator from the even_group. */ if ((ret = MPI_Comm_create(test_comm, even_group, &even_comm))) ERR(ret); /* Learn my rank and the total number of processors in even group. */ if (even_comm != MPI_COMM_NULL) { if ((ret = MPI_Comm_rank(even_comm, &even_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(even_comm, &even_size))) MPIERR(ret); } /* Create a group with tasks 0, 1, and 3. */ int overlap_ranges[OVERLAP_NUM_RANGES][3] = {{0, 0, 1}, {1, 3, 2}}; if ((ret = MPI_Group_range_incl(world_group, OVERLAP_NUM_RANGES, overlap_ranges, &overlap_group))) ERR(ret); /* Create a communicator from the overlap_group. */ if ((ret = MPI_Comm_create(test_comm, overlap_group, &overlap_comm))) ERR(ret); /* Learn my rank and the total number of processors in overlap * group. */ if (overlap_comm != MPI_COMM_NULL) { if ((ret = MPI_Comm_rank(overlap_comm, &overlap_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(overlap_comm, &overlap_size))) MPIERR(ret); } /* Initialize PIO system for even. */ if (even_comm != MPI_COMM_NULL) { if ((ret = PIOc_Init_Intracomm(even_comm, NUM_IO1, STRIDE1, BASE1, rearranger[r], &even_iosysid))) ERR(ret); /* These should not work. */ if (PIOc_set_hint(even_iosysid + TEST_VAL_42, NULL, NULL) != PIO_EBADID) ERR(ERR_WRONG); if (PIOc_set_hint(even_iosysid, NULL, NULL) != PIO_EINVAL) ERR(ERR_WRONG); /* Set the hint (which will be ignored). */ if ((ret = PIOc_set_hint(even_iosysid, "hint", "hint_value"))) ERR(ret); /* Set the error handler. */ /*PIOc_Set_IOSystem_Error_Handling(even_iosysid, PIO_BCAST_ERROR);*/ if ((ret = PIOc_set_iosystem_error_handling(even_iosysid, PIO_BCAST_ERROR, NULL))) ERR(ret); } /* Initialize PIO system for overlap comm. */ if (overlap_comm != MPI_COMM_NULL) { if ((ret = PIOc_Init_Intracomm(overlap_comm, NUM_IO2, STRIDE1, BASE1, rearranger[r], &overlap_iosysid))) ERR(ret); /* Set the error handler. */ PIOc_Set_IOSystem_Error_Handling(overlap_iosysid, PIO_BCAST_ERROR); } for (int i = 0; i < num_flavors; i++) { char fname0[PIO_MAX_NAME + 1]; char fname1[PIO_MAX_NAME + 1]; char fname2[PIO_MAX_NAME + 1]; sprintf(fname0, "%s_file_0_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[i], rearranger[r]); if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname0, ATTNAME, DIMNAME, my_rank))) ERR(ret); sprintf(fname1, "%s_file_1_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[i], rearranger[r]); if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname1, ATTNAME, DIMNAME, my_rank))) ERR(ret); sprintf(fname2, "%s_file_2_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[i], rearranger[r]); if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname2, ATTNAME, DIMNAME, my_rank))) ERR(ret); /* Now check the first file from WORLD communicator. */ int ncid; if ((ret = open_and_check_file(test_comm, iosysid_world, flavor[i], &ncid, fname0, ATTNAME, DIMNAME, 1, my_rank))) ERR(ret); /* Now have the even communicators check the files. */ int ncid2; if (even_comm != MPI_COMM_NULL) { if ((ret = open_and_check_file(even_comm, even_iosysid, flavor[i], &ncid2, fname2, ATTNAME, DIMNAME, 1, my_rank))) ERR(ret); if ((ret = check_file(even_comm, even_iosysid, flavor[i], ncid2, fname2, ATTNAME, DIMNAME, my_rank))) ERR(ret); } /* Now have the overlap communicators check the files. */ int ncid3; if (overlap_comm != MPI_COMM_NULL) { if ((ret = open_and_check_file(overlap_comm, overlap_iosysid, flavor[i], &ncid3, fname1, ATTNAME, DIMNAME, 1, my_rank))) ERR(ret); if ((ret = check_file(overlap_comm, overlap_iosysid, flavor[i], ncid3, fname1, ATTNAME, DIMNAME, my_rank))) ERR(ret); } /* Close the still-open files. */ if (even_comm != MPI_COMM_NULL) if ((ret = PIOc_closefile(ncid2))) ERR(ret); if (overlap_comm != MPI_COMM_NULL) if ((ret = PIOc_closefile(ncid3))) ERR(ret); if ((ret = PIOc_closefile(ncid))) ERR(ret); } /* next iotype */ /* Finalize PIO systems. */ if (even_comm != MPI_COMM_NULL) if ((ret = PIOc_finalize(even_iosysid))) ERR(ret); if (overlap_comm != MPI_COMM_NULL) { if ((ret = PIOc_finalize(overlap_iosysid))) ERR(ret); } if ((ret = PIOc_finalize(iosysid_world))) ERR(ret); /* Free MPI resources used by test. */ if ((ret = MPI_Group_free(&overlap_group))) ERR(ret); if ((ret = MPI_Group_free(&even_group))) ERR(ret); if ((ret = MPI_Group_free(&world_group))) ERR(ret); if (overlap_comm != MPI_COMM_NULL) if ((ret = MPI_Comm_free(&overlap_comm))) ERR(ret); if (even_comm != MPI_COMM_NULL) if ((ret = MPI_Comm_free(&even_comm))) ERR(ret); } /* next rearranger */ } /* my_rank < TARGET_NTASKS */ /* Finalize test. */ if ((ret = pio_test_finalize(&test_comm))) return ERR_AWFUL; printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); return 0; }
/** Run Tests for NetCDF-4 Functions. * * @param argc argument count * @param argv array of arguments */ int main(int argc, char **argv) { int verbose = 1; /** Zero-based rank of processor. */ int my_rank; /** Number of processors involved in current execution. */ int ntasks; /** Specifies the flavor of netCDF output format. */ int iotype; /** Different output flavors. */ int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P}; /** Names for the output files. */ char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc", "test_nc4_classic.nc", "test_nc4_serial4.nc", "test_nc4_parallel4.nc"}; /** Number of processors that will do IO. In this test we * will do IO from all processors. */ int niotasks; /** Stride in the mpi rank between io tasks. Always 1 in this * test. */ int ioproc_stride = 1; /** Number of the aggregator? Always 0 in this test. */ int numAggregator = 0; /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; /** The dimension IDs. */ int dimids[NDIM]; /** Array index per processing unit. */ PIO_Offset elements_per_pe; /** The ID for the parallel I/O system. */ int iosysid; /** The ncid of the netCDF file. */ int ncid = 0; /** The ID of the netCDF varable. */ int varid; /** Storage of netCDF-4 files (contiguous vs. chunked). */ int storage; /** Chunksizes set in the file. */ PIO_Offset my_chunksize[NDIM]; /** The shuffle filter setting in the netCDF-4 test file. */ int shuffle; /** Non-zero if deflate set for the variable in the netCDF-4 test file. */ int deflate; /** The deflate level set for the variable in the netCDF-4 test file. */ int deflate_level; /** Endianness of variable. */ int endianness; /* Size of the var chunk cache. */ PIO_Offset var_cache_size; /* Number of elements in var cache. */ PIO_Offset var_cache_nelems; /* Var cache preemption. */ float var_cache_preemption; /** The I/O description ID. */ int ioid; /** A buffer for sample data. */ float *buffer; /** A buffer for reading data back from the file. */ int *read_buffer; /** The decomposition mapping. */ PIO_Offset *compdof; /** Return code. */ int ret; /** Index for loops. */ int fmt, d, d1, i; /** For setting the chunk cache. */ PIO_Offset chunk_cache_size = 1024*1024; PIO_Offset chunk_cache_nelems = 1024; float chunk_cache_preemption = 0.5; /* For reading the chunk cache. */ PIO_Offset chunk_cache_size_in; PIO_Offset chunk_cache_nelems_in; float chunk_cache_preemption_in; char varname[15]; #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || ntasks == 8 || ntasks == 16)) fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); if (verbose) printf("%d: ParallelIO Library test_nc4 running on %d processors.\n", my_rank, ntasks); /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, ioproc_start, PIO_REARR_SUBSET, &iosysid))) ERR(ret); /* Describe the decomposition. This is a 1-based array, so add 1! */ elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (i = 0; i < elements_per_pe; i++) { compdof[i] = my_rank * elements_per_pe + i + 1; } /* Create the PIO decomposition for this test. */ if (verbose) printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, NULL, NULL))) ERR(ret); free(compdof); #ifdef HAVE_MPE /* Log with MPE that we are done with INIT. */ if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) MPIERR(ret); #endif /* HAVE_MPE */ /* How many flavors will we be running for? */ int num_flavors = 0; int fmtidx = 0; #ifdef _PNETCDF num_flavors++; format[fmtidx++] = PIO_IOTYPE_PNETCDF; #endif #ifdef _NETCDF num_flavors++; format[fmtidx++] = PIO_IOTYPE_NETCDF; #endif #ifdef _NETCDF4 num_flavors += 2; format[fmtidx++] = PIO_IOTYPE_NETCDF4C; format[fmtidx] = PIO_IOTYPE_NETCDF4P; #endif /* Use PIO to create the example file in each of the four * available ways. */ for (fmt = 0; fmt < num_flavors; fmt++) { #ifdef HAVE_MPE /* Log with MPE that we are starting CREATE. */ if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create"))) MPIERR(ret); #endif /* HAVE_MPE */ if (verbose) printf("rank: %d Setting chunk cache for file %s with format %d...\n", my_rank, filename[fmt], format[fmt]); /* Try to set the chunk cache with invalid preemption to check error handling. */ chunk_cache_preemption = 50.0; ret = PIOc_set_chunk_cache(iosysid, format[fmt], chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption); if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) { if (ret != NC_EINVAL) ERR(ERR_AWFUL); } else { if (ret != NC_ENOTNC4) ERR(ERR_AWFUL); } /* Try to set the chunk cache. */ chunk_cache_preemption = 0.5; ret = PIOc_set_chunk_cache(iosysid, format[fmt], chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption); /* Should only have worked for netCDF-4 iotypes. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) { if (ret != PIO_NOERR) ERR(ret); } else { if (ret != PIO_ENOTNC4) ERR(ERR_AWFUL); } /* Now check the chunk cache. */ ret = PIOc_get_chunk_cache(iosysid, format[fmt], &chunk_cache_size_in, &chunk_cache_nelems_in, &chunk_cache_preemption_in); /* Should only have worked for netCDF-4 iotypes. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) { /* Check that there was no error. */ if (ret != PIO_NOERR) ERR(ret); /* Check that we got the correct values. */ if (chunk_cache_size_in != chunk_cache_size || chunk_cache_nelems_in != chunk_cache_nelems || chunk_cache_preemption_in != chunk_cache_preemption) ERR(ERR_AWFUL); } else { if (ret != PIO_ENOTNC4) ERR(ERR_AWFUL); } /* Create the netCDF output file. */ if (verbose) printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename[fmt], format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], PIO_CLOBBER))) ERR(ret); /* Set error handling. */ PIOc_Set_File_Error_Handling(ncid, PIO_BCAST_ERROR); /* Define netCDF dimensions and variable. */ if (verbose) printf("rank: %d Defining netCDF metadata...\n", my_rank); for (d = 0; d < NDIM; d++) { if (verbose) printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, dim_name[d], dim_len[d]); if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); } if (verbose) printf("rank: %d Defining netCDF variable %s, ndims %d\n", my_rank, VAR_NAME, NDIM); if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); /* For netCDF-4 files, set the chunksize to improve performance. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) { if (verbose) printf("rank: %d Defining chunksizes\n", my_rank); if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize))) ERR(ret); /** Check that the inq_varname function works. */ if (verbose) printf("rank: %d Checking varname\n", my_rank); ret = PIOc_inq_varname(ncid, 0, varname); printf("rank: %d ret: %d varname: %s\n", my_rank, ret, varname); /** Check that the inq_var_chunking function works. */ if (verbose) printf("rank: %d Checking chunksizes\n"); if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize))) ERR(ret); if (verbose) { printf("rank: %d ret: %d storage: %d\n", my_rank, ret, storage); for (d1 = 0; d1 < NDIM; d1++) { printf("chunksize[%d]=%d\n", d1, my_chunksize[d1]); } } /** Check the answers. */ if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) { if (storage != NC_CHUNKED) ERR(ERR_AWFUL); for (d1 = 0; d1 < NDIM; d1++) if (my_chunksize[d1] != chunksize[d1]) ERR(ERR_AWFUL); } /* Check that the inq_var_deflate functions works. */ if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level))) ERR(ret); /** For serial netCDF-4 deflate is turned on by default */ if (format[fmt] == PIO_IOTYPE_NETCDF4C) if (shuffle || !deflate || deflate_level != 1) ERR(ERR_AWFUL); /* For parallel netCDF-4, no compression available. :-( */ if (format[fmt] == PIO_IOTYPE_NETCDF4P) if (shuffle || deflate) ERR(ERR_AWFUL); /* Check setting the chunk cache for the variable. */ printf("rank: %d PIOc_set_var_chunk_cache...\n", my_rank); if ((ret = PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS, VAR_CACHE_PREEMPTION))) ERR(ret); /* Check getting the chunk cache values for the variable. */ printf("rank: %d PIOc_get_var_chunk_cache...\n", my_rank); if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems, &var_cache_preemption))) ERR(ret); PIO_Offset len; if ((ret = PIOc_inq_dimlen(ncid, 0, &len))) ERR(ret); /* Check that we got expected values. */ printf("rank: %d var_cache_size = %d\n", my_rank, var_cache_size); if (var_cache_size != VAR_CACHE_SIZE) ERR(ERR_AWFUL); if (var_cache_nelems != VAR_CACHE_NELEMS) ERR(ERR_AWFUL); if (var_cache_preemption != VAR_CACHE_PREEMPTION) ERR(ERR_AWFUL); } else { /* Trying to set or inq netCDF-4 settings for non-netCDF-4 * files results in the PIO_ENOTNC4 error. */ if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)) != PIO_ENOTNC4) ERR(ERR_AWFUL); if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize)) != PIO_ENOTNC4) ERR(ERR_AWFUL); if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level)) != PIO_ENOTNC4) ERR(ret); if ((ret = PIOc_def_var_endian(ncid, 0, 1)) != PIO_ENOTNC4) ERR(ret); if ((ret = PIOc_inq_var_endian(ncid, 0, &endianness)) != PIO_ENOTNC4) ERR(ret); if ((ret = PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS, VAR_CACHE_PREEMPTION)) != PIO_ENOTNC4) ERR(ret); if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems, &var_cache_preemption)) != PIO_ENOTNC4) ERR(ret); if ((ret = PIOc_set_chunk_cache(iosysid, format[fmt], chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)) != PIO_ENOTNC4) ERR(ret); if ((ret = PIOc_get_chunk_cache(iosysid, format[fmt], &chunk_cache_size, &chunk_cache_nelems, &chunk_cache_preemption)) != PIO_ENOTNC4) ERR(ret); } if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Close the netCDF file. */ if (verbose) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); } /* Free the PIO decomposition. */ if (verbose) printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ if (verbose) printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif return 0; }
/* Write, then read, a simple example with darrays. The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump): <pre> netcdf darray_no_async_iotype_1 { dimensions: unlimted = UNLIMITED ; // (2 currently) x = 4 ; y = 4 ; variables: int foo(unlimted, x, y) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45, 142, 142, 142, 142, 143, 143, 143, 143, 144, 144, 144, 144, 145, 145, 145, 145 ; } </pre> */ int main(int argc, char* argv[]) { int my_rank; /* Zero-based rank of processor. */ int ntasks; /* Number of processors involved in current execution. */ int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ int ioproc_start = 0; /* Rank of first task to be used for I/O. */ PIO_Offset elements_per_pe; /* Array elements per processing unit. */ int iosysid; /* The ID for the parallel I/O system. */ int ncid; /* The ncid of the netCDF file. */ int dimid[NDIM3]; /* The dimension ID. */ int varid; /* The ID of the netCDF varable. */ int ioid; /* The I/O description ID. */ char filename[NC_MAX_NAME + 1]; /* Test filename. */ int num_flavors = 0; /* Number of iotypes available in this build. */ int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */ int ret; /* Return value. */ #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ if (ntasks != TARGET_NTASKS) fprintf(stderr, "Number of processors must be 4!\n"); printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n", my_rank, ntasks); /* Turn on logging. */ if ((ret = PIOc_set_log_level(LOG_LEVEL))) return ret; /* Initialize the PIO IO system. This specifies how many and * which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride, ioproc_start, PIO_REARR_BOX, &iosysid))) ERR(ret); /* Describe the decomposition. */ elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS; /* Allocate and initialize array of decomposition mapping. */ PIO_Offset compdof[elements_per_pe]; for (int i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i; /* Create the PIO decomposition for this example. Since this * is a variable with an unlimited dimension, we want to * create a 2-D composition which represents one record. */ printf("rank: %d Creating decomposition...\n", my_rank); if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, compdof, &ioid, 0, NULL, NULL))) ERR(ret); /* The number of favors may change with the build parameters. */ #ifdef _PNETCDF format[num_flavors++] = PIO_IOTYPE_PNETCDF; #endif format[num_flavors++] = PIO_IOTYPE_NETCDF; #ifdef _NETCDF4 format[num_flavors++] = PIO_IOTYPE_NETCDF4C; format[num_flavors++] = PIO_IOTYPE_NETCDF4P; #endif /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* Create a filename. */ sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); /* Create the netCDF output file. */ printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename, format[fmt]); if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) ERR(ret); /* Define netCDF dimension and variable. */ printf("rank: %d Defining netCDF metadata...\n", my_rank); for (int d = 0; d < NDIM3; d++) if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) ERR(ret); if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) ERR(ret); if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Allocate storage for sample data. */ int buffer[elements_per_pe]; /* Write each timestep. */ for (int t = 0; t < NUM_TIMESTEPS; t++) { /* Create some data for this timestep. */ for (int i = 0; i < elements_per_pe; i++) buffer[i] = 100 * t + START_DATA_VAL + my_rank; /* Write data to the file. */ printf("rank: %d Writing sample data...\n", my_rank); if ((ret = PIOc_setframe(ncid, varid, t))) ERR(ret); if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) ERR(ret); } /* THis will cause all data to be written to disk. */ if ((ret = PIOc_sync(ncid))) ERR(ret); /* Close the netCDF file. */ printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); /* Check the output file. */ /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */ /* my_rank, ioid))) */ /* ERR(ret); */ } /* Free the PIO decomposition. */ printf("rank: %d Freeing PIO decomposition...\n", my_rank); if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); /* Finalize the IO system. */ printf("rank: %d Freeing PIO resources...\n", my_rank); if ((ret = PIOc_finalize(iosysid))) ERR(ret); /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif printf("rank: %d SUCCESS!\n", my_rank); return 0; }
/* Write, then read, a simple example with darrays. The sample file created by this program is a small netCDF file. It has the following contents (as shown by ncdump): <pre> netcdf darray_no_async_iotype_1 { dimensions: unlimted = UNLIMITED ; // (2 currently) x = 4 ; y = 4 ; variables: int foo(unlimted, x, y) ; data: foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45, 142, 142, 142, 142, 143, 143, 143, 143, 144, 144, 144, 144, 145, 145, 145, 145 ; } </pre> */ int main(int argc, char* argv[]) { int my_rank; /* Zero-based rank of processor. */ int ntasks; /* Number of processors involved in current execution. */ int iosysid; /* The ID for the parallel I/O system. */ /* int ncid; /\* The ncid of the netCDF file. *\/ */ /* int dimid[NDIM3]; /\* The dimension ID. *\/ */ /* int varid; /\* The ID of the netCDF varable. *\/ */ /* char filename[NC_MAX_NAME + 1]; /\* Test filename. *\/ */ /* int num_flavors = 0; /\* Number of iotypes available in this build. *\/ */ /* int format[NUM_NETCDF_FLAVORS]; /\* Different output flavors. *\/ */ int ret; /* Return value. */ #ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; #endif /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) MPIERR(ret); if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) MPIERR(ret); /* Check that a valid number of processors was specified. */ printf("%d: ParallelIO Library darray_async example running on %d processors.\n", my_rank, ntasks); if (ntasks != TARGET_NTASKS) { fprintf(stderr, "Number of processors must be %d!\n", TARGET_NTASKS); return ERR_BAD; } /* Turn on logging. */ if ((ret = PIOc_set_log_level(LOG_LEVEL))) return ret; /* Num procs for computation. */ int num_procs2[COMPONENT_COUNT] = {4}; /* Is the current process a computation task? */ int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1; /* Initialize the IO system. */ if ((ret = PIOc_init_async(MPI_COMM_WORLD, NUM_IO_TASKS, NULL, COMPONENT_COUNT, num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid))) ERR(ret); /* The rest of the code executes on computation tasks only. As * PIO functions are called on the computation tasks, the * async system will call them on the IO task. When the * computation tasks call PIO_finalize(), the IO task will get * a message to shut itself down. */ if (comp_task) { /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */ /* int ioid; /\* The I/O description ID. *\/ */ /* /\* How many elements on each computation task? *\/ */ /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */ /* /\* Allocate and initialize array of decomposition mapping. *\/ */ /* PIO_Offset compdof[elements_per_pe]; */ /* for (int i = 0; i < elements_per_pe; i++) */ /* compdof[i] = my_rank * elements_per_pe + i; */ /* /\* Create the PIO decomposition for this example. Since */ /* this is a variable with an unlimited dimension, we want */ /* to create a 2-D composition which represents one */ /* record. *\/ */ /* printf("rank: %d Creating decomposition...\n", my_rank); */ /* if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, */ /* compdof, &ioid, 0, NULL, NULL))) */ /* ERR(ret); */ /* /\* The number of favors may change with the build parameters. *\/ */ /* #ifdef _PNETCDF */ /* format[num_flavors++] = PIO_IOTYPE_PNETCDF; */ /* #endif */ /* format[num_flavors++] = PIO_IOTYPE_NETCDF; */ /* #ifdef _NETCDF4 */ /* format[num_flavors++] = PIO_IOTYPE_NETCDF4C; */ /* format[num_flavors++] = PIO_IOTYPE_NETCDF4P; */ /* #endif */ /* /\* Use PIO to create the example file in each of the four */ /* * available ways. *\/ */ /* for (int fmt = 0; fmt < num_flavors; fmt++) */ /* { */ /* /\* Create a filename. *\/ */ /* sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); */ /* /\* Create the netCDF output file. *\/ */ /* printf("rank: %d Creating sample file %s with format %d...\n", */ /* my_rank, filename, format[fmt]); */ /* if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) */ /* ERR(ret); */ /* /\* Define netCDF dimension and variable. *\/ */ /* printf("rank: %d Defining netCDF metadata...\n", my_rank); */ /* for (int d = 0; d < NDIM3; d++) */ /* if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) */ /* ERR(ret); */ /* if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) */ /* ERR(ret); */ /* if ((ret = PIOc_enddef(ncid))) */ /* ERR(ret); */ /* /\* Allocate storage for sample data. *\/ */ /* int buffer[elements_per_pe]; */ /* /\* Write each timestep. *\/ */ /* for (int t = 0; t < NUM_TIMESTEPS; t++) */ /* { */ /* /\* Create some data for this timestep. *\/ */ /* for (int i = 0; i < elements_per_pe; i++) */ /* buffer[i] = 100 * t + START_DATA_VAL + my_rank; */ /* /\* Write data to the file. *\/ */ /* printf("rank: %d Writing sample data...\n", my_rank); */ /* if ((ret = PIOc_setframe(ncid, varid, t))) */ /* ERR(ret); */ /* if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) */ /* ERR(ret); */ /* } */ /* /\* THis will cause all data to be written to disk. *\/ */ /* if ((ret = PIOc_sync(ncid))) */ /* ERR(ret); */ /* /\* Close the netCDF file. *\/ */ /* printf("rank: %d Closing the sample data file...\n", my_rank); */ /* if ((ret = PIOc_closefile(ncid))) */ /* ERR(ret); */ /* /\* Check the output file. *\/ */ /* /\* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, *\/ */ /* /\* my_rank, ioid))) *\/ */ /* /\* ERR(ret); *\/ */ /* } */ /* Free the PIO decomposition. */ /* printf("rank: %d Freeing PIO decomposition...\n", my_rank); */ /* if ((ret = PIOc_freedecomp(iosysid, ioid))) */ /* ERR(ret); */ /* Finalize the IO system. Only call this from the computation tasks. */ printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME); if ((ret = PIOc_finalize(iosysid))) ERR(ret); } /* endif comp_task */ /* Finalize the MPI library. */ MPI_Finalize(); #ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; #endif printf("rank: %d SUCCESS!\n", my_rank); return 0; }