/* Test a small file with two record vars, which grow, and has * attributes added. */ static int test_two_growing_with_att(const char *testfile, int cmode) { int err, ncid, dimid, varid[NUM_VARS]; char data[MAX_RECS], data_in; char att_name[NC_MAX_NAME + 1]; MPI_Offset start[ONE_DIM], count[ONE_DIM], index[ONE_DIM], len_in; int v, r; /* Create a file with one ulimited dimensions, and one var. */ err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid); ERR err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid[0]); ERR err=ncmpi_def_var(ncid, VAR_NAME2, NC_CHAR, 1, &dimid, &varid[1]); ERR err=ncmpi_close(ncid); ERR /* Create some phoney data. */ for (data[0] = 'a', r = 1; r < MAX_RECS; r++) data[r] = data[r - 1] + 1; /* Normally one would not close and reopen the file for each * record, nor add an attribute each time I add a record, but I am * giving the library a little work-out here... */ for (r = 0; r < MAX_RECS; r++) { /* Write one record of var data, a single character. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_WRITE, MPI_INFO_NULL, &ncid); ERR count[0] = 1; start[0] = r; sprintf(att_name, "a_%d", data[r]); for (v = 0; v < NUM_VARS; v++) { err=ncmpi_put_vara_text_all(ncid, varid[v], start, count, &data[r]); ERR err=ncmpi_redef(ncid); ERR err=ncmpi_put_att_text(ncid, varid[v], att_name, 1, &data[r]); ERR err=ncmpi_enddef(ncid); ERR } err=ncmpi_close(ncid); ERR /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq_dimlen(ncid, 0, &len_in); ERR if (len_in != r + 1) {printf("Error at line %d\n",__LINE__);return 1;} index[0] = r; err=ncmpi_begin_indep_data(ncid); ERR for (v = 0; v < NUM_VARS; v++) { err=ncmpi_get_var1_text(ncid, varid[v], index, &data_in); ERR if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;} } err=ncmpi_close(ncid); ERR } /* Next record. */ return 0; }
/* Test a small file with one var and one att. */ static int test_one_with_att(const char *testfile, int cmode) { int err, ncid, dimid, varid; char data = 'h', data_in; int ndims, nvars, natts, unlimdimid; MPI_Offset start[NDIMS], count[NDIMS]; /* Create a file with one ulimited dimensions, and one var. */ err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid); ERR err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid); ERR err=ncmpi_put_att_text(ncid, NC_GLOBAL, ATT_NAME, 1, &data); ERR err=ncmpi_enddef(ncid); ERR /* Write one record of var data, a single character. */ count[0] = 1; start[0] = 0; err=ncmpi_put_vara_text_all(ncid, varid, start, count, &data); ERR /* We're done! */ err=ncmpi_close(ncid); ERR /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_var_text_all(ncid, varid, &data_in); ERR if (data_in != data) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_att_text(ncid, NC_GLOBAL, ATT_NAME, &data_in); ERR if (data_in != data) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR return 0; }
/* Test a small file with one record var, which grows. */ static int test_one_growing(const char *testfile, int cmode) { int err, ncid, dimid, varid; char data[MAX_RECS], data_in; MPI_Offset start[ONE_DIM], count[ONE_DIM], index[ONE_DIM], len_in; int r, f; /* Create some phoney data. */ for (data[0] = 'a', r = 1; r < MAX_RECS; r++) data[r] = data[r - 1] + 1; /* Run this with and without fill mode. */ for (f = 0; f < 2; f++) { /* Create a file with one ulimited dimensions, and one var. */ err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid); ERR err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid); ERR err=ncmpi_close(ncid); ERR /* Normally one would not close and reopen the file for each * record, but I am giving the library a little work-out here... */ for (r = 0; r < MAX_RECS; r++) { /* Write one record of var data, a single character. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_WRITE, MPI_INFO_NULL, &ncid); ERR /* if (f) { err=ncmpi_set_fill(ncid, NC_NOFILL, NULL); ERR} */ count[0] = 1; start[0] = r; err=ncmpi_put_vara_text_all(ncid, varid, start, count, &data[r]); ERR err=ncmpi_close(ncid); ERR /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq_dimlen(ncid, 0, &len_in); ERR if (len_in != r + 1) {printf("Error at line %d\n",__LINE__);return 1;} index[0] = r; err=ncmpi_begin_indep_data(ncid); ERR err=ncmpi_get_var1_text(ncid, 0, index, &data_in); ERR if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR } /* Next record. */ } return 0; }
int main(int argc, char** argv) { extern int optind; char *filename="testfile.nc"; int i, rank, verbose=1, err; int ncid, cmode, omode; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); /* get command-line arguments */ while ((i = getopt(argc, argv, "hq")) != EOF) switch(i) { case 'q': verbose = 0; break; case 'h': default: if (rank==0) usage(argv[0]); MPI_Finalize(); return 0; } argc -= optind; argv += optind; if (argc == 1) filename = argv[0]; /* optional argument */ if (verbose && rank == 0) printf("%s: example of file create and open\n",__FILE__); /* create a new file using clobber mode ----------------------------------*/ cmode = NC_CLOBBER; err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); ERR /* close file */ err = ncmpi_close(ncid); ERR /* open the newly created file for read only -----------------------------*/ omode = NC_NOWRITE; err = ncmpi_open(MPI_COMM_WORLD, filename, omode, MPI_INFO_NULL, &ncid); ERR /* close file */ err = ncmpi_close(ncid); ERR /* check if there is any PnetCDF internal malloc residue */ MPI_Offset malloc_size, sum_size; err = ncmpi_inq_malloc_size(&malloc_size); if (err == NC_NOERR) { MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && sum_size > 0) printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", sum_size); } MPI_Finalize(); return 0; }
int Traj_AmberNetcdf::parallelOpenTrajout(Parallel::Comm const& commIn) { if (Ncid() != -1) return 0; int err = ncmpi_open(commIn.MPIcomm(), filename_.full(), NC_WRITE, MPI_INFO_NULL, &ncid_); if (checkPNCerr(err)) { mprinterr("Error: Opening NetCDF file '%s' for writing in parallel.\n", filename_.full()); return 1; } err = ncmpi_begin_indep_data( ncid_ ); // Independent data mode return 0; }
static int NC5_open(const char *path, int cmode, int basepe, size_t *chunksizehintp, int use_parallel, void* mpidata, struct NC_Dispatch* table, NC* nc) { int res; NC5_INFO* nc5; MPI_Comm comm = MPI_COMM_WORLD; MPI_Info info = MPI_INFO_NULL; /* Check the cmode for only valid flags*/ if(cmode & ~LEGAL_OPEN_FLAGS) return NC_EINVAL; /* Cannot have both MPIO flags */ if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX)) return NC_EINVAL; /* Appears that this comment is wrong; allow 64 bit offset*/ /* Cannot have 64 bit offset flag */ /* if(cmode & (NC_64BIT_OFFSET)) return NC_EINVAL; */ if(mpidata != NULL) { comm = ((NC_MPI_INFO *)mpidata)->comm; info = ((NC_MPI_INFO *)mpidata)->info; } else { comm = MPI_COMM_WORLD; info = MPI_INFO_NULL; } /* Fix up the cmode by keeping only essential flags; these are the flags that are the same in netcf.h and pnetcdf.h */ cmode &= (NC_WRITE | NC_NOCLOBBER | NC_LOCK | NC_SHARE | NC_64BIT_OFFSET); cmode |= (NC_NETCDF4); /* see comment in NC5_create */ /* Create our specific NC5_INFO instance */ nc5 = (NC5_INFO*)calloc(1,sizeof(NC5_INFO)); if(nc5 == NULL) return NC_ENOMEM; /* Link nc5 and nc */ NC5_DATA_SET(nc,nc5); res = ncmpi_open(comm, path, cmode, info, &(nc->int_ncid)); /* Default to independent access, like netCDF-4/HDF5 files. */ if(!res) { res = ncmpi_begin_indep_data(nc->int_ncid); nc5->pnetcdf_access_mode = NC_INDEPENDENT; } return res; }
//---------------------------------------------------------------- // Open the first NETCDF file and query all the types for all the variables static void determine_var_types() { assert(netcdf_file_names != 0); assert(netcdf_var_names != 0); struct Type type; int plist_id,i = 0,varidp,ndimsp; int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[0], NC_NOWRITE, MPI_INFO_NULL, &plist_id); if (file_id != NC_NOERR) terminate_with_error_msg("ERROR: Cannot open file %s\n", netcdf_file_names[0]); var_types = (struct Type *)calloc(var_count, sizeof(*var_types)); nc_type xtypep; for (i = 0; i < var_count; ++i) { int dataset_id = ncmpi_inq_varid (plist_id, netcdf_var_names[i], &varidp); dataset_id = ncmpi_inq_vartype(plist_id,varidp, &xtypep); if (dataset_id != NC_NOERR) terminate_with_error_msg("ERROR: Cannot read the datatype of the variable %s\n", netcdf_var_names[i]); int num_dims = ncmpi_inq_varndims(plist_id,varidp,&ndimsp); if (ndimsp > 3 ) { //TODO:probably we can make it more clever to handle more dimensions as they are not related to the variable itself on netcdf type.atomic_type = INVALID; // we don't support arrays of more than 3 dimension return; } else if (xtypep == NC_FLOAT || xtypep == NC_INT) { type.num_values = 1; } else // we don't support HD5_COMPOUND datatype for example { type.atomic_type = INVALID; } type.atomic_type = from_netcdf_atomic_type(xtypep); var_types[i] = type; if (var_types[i].atomic_type == INVALID) terminate_with_error_msg("ERROR: The datatype of the %s variable is not supported\n", netcdf_var_names[i]); } ncmpi_close(plist_id); }
// Traj_NcEnsemble::openTrajin() int Traj_NcEnsemble::openTrajin() { // If already open, return if (Ncid()!=-1) return 0; # if HAS_PNETCDF int err = ncmpi_open(MPI_COMM_WORLD, filename_.full(), NC_NOWRITE, MPI_INFO_NULL, &ncid_); //err += ncmpi_begin_indep_data( ncid_ ); // Disable independent data mode # else int err = NC_openRead( filename_.Full() ); # endif if ( err != 0 ) { mprinterr("Error: Opening Netcdf file %s for reading.\n", filename_.base()); return 1; } return 0; }
/* Test a small file with an unlimited dimension. NOTE: Normally I * write a NULL terminator for my attributes and text strings, but * this reproduces a bug that a fortran user sent us. So string data * are written to the file without null terminators. - Ed */ static int test_small_unlim(const char *testfile, int cmode) { int i, err, ncid, dimids[NDIMS], varid; char data[NUM_VALS][STR_LEN + 1], data_in[NUM_VALS][STR_LEN]; int ndims, nvars, natts, unlimdimid; MPI_Offset start[NDIMS], count[NDIMS]; /* Create null-terminated text strings of correct length. */ /*for (i = 0; i < NUM_VALS; i++) strcpy(data[i], source);*/ strcpy(data[0], "2005-04-11_12:00:00"); strcpy(data[1], "2005-04-11_13:00:00"); /* Create a file with two dimensions, one unlimited, and one * var, and a global att. */ err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, dimids); ERR err=ncmpi_def_dim(ncid, DIM2_NAME, STR_LEN, &dimids[1]); ERR err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 2, dimids, &varid); ERR err=ncmpi_put_att_text(ncid, NC_GLOBAL, ATT_NAME2, strlen(TITLE), TITLE); ERR err=ncmpi_enddef(ncid); ERR /* Write some records of var data. */ count[0] = 1; count[1] = STR_LEN; start[1] = 0; for (start[0] = 0; start[0] < NUM_VALS; start[0]++) { err=ncmpi_put_vara_text_all(ncid, varid, start, count, data[start[0]]); ERR } /* We're done! */ err=ncmpi_close(ncid); ERR /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR if (ndims != 2 && nvars != 1 && natts != 0 && unlimdimid != 0) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_var_text_all(ncid, varid, (char *)data_in); ERR for (i = 0; i < NUM_VALS; i++) /* if (strncmp(data[i], data_in[i], STR_LEN)) {printf("Error at line %d\n",__LINE__);return 1;} */ if (strncmp(data[i], data_in[i], STR_LEN)) { printf("i=%d data=%s data_in=%s\n",i,data[i],data_in[i]); } err=ncmpi_close(ncid); ERR return 0; }
void BIL_Pio_read_nc_blocks(MPI_Comm all_readers_comm, MPI_Comm io_comm, int num_blocks, BIL_Block* blocks) { int i; for (i = 0; i < num_blocks; i++) { int fp; BIL_Timing_fopen_start(all_readers_comm); assert(ncmpi_open(io_comm, blocks[i].file_name, NC_NOWRITE, BIL->io_hints, &fp) == NC_NOERR); BIL_Timing_fopen_stop(all_readers_comm); ncmpi_begin_indep_data(fp); // Find the id, type, and size of the variable. int var_id; assert(ncmpi_inq_varid(fp, blocks[i].var_name, &var_id) == NC_NOERR); nc_type var_type; assert(ncmpi_inq_vartype(fp, var_id, &var_type) == NC_NOERR); // Create extra variables specifically for the netCDF API. MPI_Offset nc_dim_starts[BIL_MAX_NUM_DIMS]; MPI_Offset nc_dim_sizes[BIL_MAX_NUM_DIMS]; int j; for (j = 0; j < blocks[i].num_dims; j++) { nc_dim_starts[j] = blocks[i].starts[j]; nc_dim_sizes[j] = blocks[i].sizes[j]; } MPI_Datatype nc_var_type; BIL_Pio_nc_to_mpi_type(var_type, &nc_var_type, &(blocks[i].var_size)); // Allocate room for data and read it independently. blocks[i].data = BIL_Misc_malloc(blocks[i].total_size * blocks[i].var_size); BIL_Timing_io_start(all_readers_comm); assert(ncmpi_get_vara(fp, var_id, nc_dim_starts, nc_dim_sizes, blocks[i].data, blocks[i].total_size, nc_var_type) == NC_NOERR); BIL_Timing_io_stop(all_readers_comm, blocks[i].total_size * blocks[i].var_size); // Clean up. ncmpi_end_indep_data(fp); ncmpi_close(fp); } }
void * IOR_Open_NCMPI(char * testFileName, IOR_param_t * param) { int * fd; int fd_mode; MPI_Info mpiHints = MPI_INFO_NULL; /* Wei-keng Liao: read and set MPI file hints from hintsFile */ SetHints(&mpiHints, param->hintsFileName); if (rank == 0 && param->showHints) { fprintf(stdout, "\nhints passed to MPI_File_open() {\n"); ShowHints(&mpiHints); fprintf(stdout, "}\n"); } fd = (int *)malloc(sizeof(int)); if (fd == NULL) ERR("Unable to malloc file descriptor"); fd_mode = GetFileMode(param); NCMPI_CHECK(ncmpi_open(testComm, testFileName, fd_mode, mpiHints, fd), "cannot open file"); /* Wei-keng Liao: print the MPI file hints currently used */ /* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release if (rank == 0 && param->showHints) { MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints), "cannot get file info"); fprintf(stdout, "\nhints returned from opened file {\n"); ShowHints(&mpiHints); fprintf(stdout, "}\n"); } */ /* Wei-keng Liao: free up the mpiHints object */ /* WEL - this needs future fix from next release of PnetCDF if (mpiHints != MPI_INFO_NULL) MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info"); */ return(fd); } /* IOR_Open_NCMPI() */
static int test_small_atts(const char *testfile, int cmode) { int ncid, err; char att[MAX_LEN + 1], att_in[MAX_LEN + 1], source[MAX_LEN + 1] = "0123456"; int ndims, nvars, natts, unlimdimid; MPI_Offset len_in; int t, f; /* Run this with and without fill mode. */ for (f = 0; f < 2; f++) { /* Create small files with an attribute that grows by one each * time. */ for (t = 1; t < MAX_LEN; t++) { /* Create null-terminated text string of correct length. */ strncpy(att, source, t); att[t] = '\0'; /* Create a file with one attribute. */ err = ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR err = ncmpi_put_att_text(ncid, NC_GLOBAL, ATT_NAME, t + 1, att); ERR if (f) { err=ncmpi_set_fill(ncid, NC_NOFILL, NULL); ERR} err=ncmpi_close(ncid); ERR; /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR if (ndims != 0 && nvars != 0 && natts != 1 && unlimdimid != -1) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &len_in); ERR if (len_in != t + 1) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_att_text(ncid, NC_GLOBAL, ATT_NAME, att_in); ERR if (strncmp(att_in, att, t)) {printf("Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR } } return 0; }
// Traj_NcEnsemble::setupTrajout() int Traj_NcEnsemble::setupTrajout(FileName const& fname, Topology* trajParm, CoordinateInfo const& cInfoIn, int NframesToWrite, bool append) { int err = 0; # ifdef MPI if (NoPnetcdf()) return 1; # endif readAccess_ = false; if (!append) { CoordinateInfo cInfo = cInfoIn; // TODO: File output modifications SetCoordInfo( cInfo ); # ifdef MPI ensembleStart_ = Parallel::World().Rank(); ensembleEnd_ = Parallel::World().Rank() + 1; # else ensembleStart_ = 0; ensembleEnd_ = cInfo.EnsembleSize();; # endif filename_ = fname; // Set up title if (Title().empty()) SetTitle("Cpptraj Generated trajectory"); # ifdef MPI if (Parallel::World().Master()) { // Only master creates file. # endif // Create NetCDF file. err = NC_create(filename_.Full(), NC_AMBERENSEMBLE, trajParm->Natom(), CoordInfo(), Title(), debug_); // Close Netcdf file. It will be reopened write. FIXME should NC_create leave it closed? NC_close(); # ifdef MPI } Parallel::World().MasterBcast(&err, 1, MPI_INT); # endif if (err != 0) return 1; # ifdef MPI // Synchronize netcdf info on non-master threads Sync(Parallel::World()); // DEBUG: Print info for all ranks DebugVIDs(); # endif // Allocate memory if (Coord_!=0) delete[] Coord_; Coord_ = new float[ Ncatom3() ]; } else { // NOTE: File existence is checked for in Trajout // Call setupTrajin to set input parameters. This will also allocate // memory for coords. if (setupTrajin(fname, trajParm) == TRAJIN_ERR) return 1; if (debug_ > 0) mprintf("\tNetCDF: Appending %s starting at frame %i\n", filename_.base(), Ncframe()); } // Open file # ifdef HAS_PNETCDF err = ncmpi_open(MPI_COMM_WORLD, filename_.full(), NC_WRITE, MPI_INFO_NULL, &ncid_); // TODO: Graceful error handling # else err = NC_openWrite( filename_.Full() ); # endif if ( err != 0 ) { mprinterr("Error: Opening Netcdf file %s for Write.\n", filename_.base()); return 1; } return 0; }
int main(int argc, char** argv) { char filename[256]; int i, j, rank, nprocs, err, nerrs=0, expected; int ncid, cmode, varid[2], dimid[2], req[4], st[4], *buf; int *buf0, *buf1, *buf2; size_t len; MPI_Offset start[2], count[2]; MPI_Info info; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); /* this program is intended to run on one process */ if (rank) goto fn_exit; /* get command-line arguments */ if (argc > 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); MPI_Finalize(); return 1; } if (argc == 2) snprintf(filename, 256, "%s", argv[1]); else strcpy(filename, "testfile.nc"); if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); sprintf(cmd_str, "*** TESTING C %s for writing interleaved fileviews ", basename(argv[0])); printf("%-66s ------ ", cmd_str); free(cmd_str); } MPI_Info_create(&info); MPI_Info_set(info, "romio_cb_write", "disable"); MPI_Info_set(info, "ind_wr_buffer_size", "8"); /* these 2 hints are required to cause a core dump if r1758 fix is not * presented */ /* create a new file for writing ----------------------------------------*/ cmode = NC_CLOBBER | NC_64BIT_DATA; err = ncmpi_create(MPI_COMM_SELF, filename, cmode, info, &ncid); CHECK_ERR MPI_Info_free(&info); /* define dimensions Y and X */ err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR /* define 2D variables of integer type */ err = ncmpi_def_var(ncid, "var0", NC_INT, 2, dimid, &varid[0]); CHECK_ERR err = ncmpi_def_var(ncid, "var1", NC_INT, 2, dimid, &varid[1]); CHECK_ERR /* enable fill mode */ err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR /* do not forget to exit define mode */ err = ncmpi_enddef(ncid); CHECK_ERR /* now we are in data mode */ buf = (int*) malloc(NY*NX * sizeof(int)); /* fill the entire variable var0 with -1s */ for (i=0; i<NY*NX; i++) buf[i] = -1; err = ncmpi_put_var_int_all(ncid, varid[0], buf); CHECK_ERR /* write 8 x 2 elements so this only interleaves the next two * iput requests */ start[0] = 0; start[1] = 3; count[0] = 8; count[1] = 2; len = (size_t)(count[0] * count[1]); buf0 = (int*) malloc(len * sizeof(int)); for (i=0; i<len; i++) buf0[i] = 50+i; err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf0, &req[0]); CHECK_ERR /* write 1 x 3 elements */ start[0] = 1; start[1] = 8; count[0] = 1; count[1] = 5; len = (size_t)(count[0] * count[1]); buf1 = (int*) malloc(len * sizeof(int)); for (i=0; i<len; i++) buf1[i] = 60+i; err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf1, &req[1]); CHECK_ERR /* write 1 x 3 elements */ start[0] = 3; start[1] = 7; count[0] = 1; count[1] = 5; len = (size_t)(count[0] * count[1]); buf2 = (int*) malloc(len * sizeof(int)); for (i=0; i<len; i++) buf2[i] = 70+i; err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf2, &req[2]); CHECK_ERR err = ncmpi_wait_all(ncid, 3, req, st); CHECK_ERR free(buf0); free(buf1); free(buf2); /* fill the entire variable var1 with -1s */ for (i=0; i<NY*NX; i++) buf[i] = -1; err = ncmpi_put_var_int_all(ncid, varid[1], buf); CHECK_ERR /* write 8 x 2 elements so this only interleaves the next two iput * requests */ start[0] = 0; start[1] = 3; count[0] = 8; count[1] = 2; len = (size_t)(count[0] * count[1]); buf0 = (int*) malloc(len * sizeof(int)); for (i=0; i<count[0]*count[1]; i++) buf0[i] = 50+i; err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf0, &req[0]); CHECK_ERR /* rearrange buffer contents, as buf is 2D */ for (i=0; i<5; i++) buf[i] = 10 + i; for (i=5; i<10; i++) buf[i] = 10 + i + 5; for (i=10; i<15; i++) buf[i] = 10 + i + 10; start[0] = 6; start[1] = 7; count[0] = 3; count[1] = 5; err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf, &req[1]); CHECK_ERR for (i=15; i<20; i++) buf[i] = 10 + i - 10; for (i=20; i<25; i++) buf[i] = 10 + i - 5; start[0] = 6; start[1] = 12; count[0] = 2; count[1] = 5; err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf+15, &req[2]); CHECK_ERR for (i=25; i<30; i++) buf[i] = 10 + i; start[0] = 8; start[1] = 12; count[0] = 1; count[1] = 5; err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf+25, &req[3]); CHECK_ERR err = ncmpi_wait_all(ncid, 4, req, st); CHECK_ERR /* check if write buffer contents have been altered */ for (i=0; i<16; i++) CHECK_CONTENTS(buf0, 50 + i) for (i=0; i<5; i++) CHECK_CONTENTS(buf, 10 + i) for (i=5; i<10; i++) CHECK_CONTENTS(buf, 10 + i + 5) for (i=10; i<15; i++) CHECK_CONTENTS(buf, 10 + i + 10) for (i=15; i<20; i++) CHECK_CONTENTS(buf, 10 + i - 10) for (i=20; i<25; i++) CHECK_CONTENTS(buf, 10 + i - 5) for (i=25; i<30; i++) CHECK_CONTENTS(buf, 10 + i) err = ncmpi_close(ncid); CHECK_ERR free(buf0); /* open the same file and read back for validate */ err = ncmpi_open(MPI_COMM_SELF, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR err = ncmpi_inq_varid(ncid, "var0", &varid[0]); CHECK_ERR err = ncmpi_inq_varid(ncid, "var1", &varid[1]); CHECK_ERR /* read the entire array */ for (i=0; i<NY*NX; i++) buf[i] = -1; err = ncmpi_get_var_int_all(ncid, varid[0], buf); CHECK_ERR /* check if the contents of buf are expected */ expected = 50; for (j=0; j<8; j++) { for (i=3; i<5; i++) { if (buf[j*NX+i] != expected) { printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n", rank, j, i, buf[j*NX+i], expected); nerrs++; } expected++; } } expected = 60; j = 1; for (i=8; i<13; i++) { if (buf[j*NX+i] != expected) { printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n", rank, j, i, buf[j*NX+i], expected); nerrs++; } expected++; } expected = 70; j = 3; for (i=7; i<12; i++) { if (buf[j*NX+i] != expected) { printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n", rank, j, i, buf[j*NX+i], expected); nerrs++; } expected++; } /* initialize the contents of the array to a different value */ for (i=0; i<NY*NX; i++) buf[i] = -1; /* read the entire array */ err = ncmpi_get_var_int_all(ncid, varid[1], buf); CHECK_ERR /* check if the contents of buf are expected */ expected = 10; for (j=6; j<9; j++) { for (i=7; i<17; i++) { if (buf[j*NX+i] != expected) { printf("%d: Unexpected read buf[%d]=%d, should be %d\n", rank, i, buf[j*NX+i], expected); nerrs++; } expected++; } } expected = 50; for (j=0; j<8; j++) { for (i=3; i<5; i++) { if (buf[j*NX+i] != expected) { printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n", rank, j, i, buf[j*NX+i], expected); nerrs++; } expected++; } } err = ncmpi_close(ncid); CHECK_ERR free(buf); /* check if PnetCDF freed all internal malloc */ MPI_Offset malloc_size; err = ncmpi_inq_malloc_size(&malloc_size); if (err == NC_NOERR && malloc_size > 0) { printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", malloc_size); ncmpi_inq_malloc_list(); } fn_exit: MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); else printf(PASS_STR); } MPI_Finalize(); return (nerrs > 0); }
static int tst_norm(char *filename, int cmode) { int ncid, dimid, varid; int dimids[NDIMS]; /* unnormalized UTF-8 encoding for Unicode 8-character "Hello" in Greek: */ unsigned char uname_utf8[] = { 0x41, /* LATIN CAPITAL LETTER A */ 0xCC, 0x80, /* COMBINING GRAVE ACCENT */ 0x41, /* LATIN CAPITAL LETTER A */ 0xCC, 0x81, /* COMBINING ACUTE ACCENT */ 0x41, /* LATIN CAPITAL LETTER A */ 0xCC, 0x82, /* COMBINING CIRCUMFLEX ACCENT */ 0x41, /* LATIN CAPITAL LETTER A */ 0xCC, 0x83, /* COMBINING TILDE */ 0x41, /* LATIN CAPITAL LETTER A */ 0xCC, 0x88, /* COMBINING DIAERESIS */ 0x41, /* LATIN CAPITAL LETTER A */ 0xCC, 0x8A, /* COMBINING RING ABOVE */ 0x43, /* LATIN CAPITAL LETTER C */ 0xCC, 0xA7, /* COMBINING CEDILLA */ 0x45, /* LATIN CAPITAL LETTER E */ 0xCC, 0x80, /* COMBINING GRAVE ACCENT */ 0x45, /* LATIN CAPITAL LETTER E */ 0xCC, 0x81, /* COMBINING ACUTE ACCENT */ 0x45, /* LATIN CAPITAL LETTER E */ 0xCC, 0x82, /* COMBINING CIRCUMFLEX ACCENT */ 0x45, /* LATIN CAPITAL LETTER E */ 0xCC, 0x88, /* COMBINING DIAERESIS */ 0x49, /* LATIN CAPITAL LETTER I */ 0xCC, 0x80, /* COMBINING GRAVE ACCENT */ 0x49, /* LATIN CAPITAL LETTER I */ 0xCC, 0x81, /* COMBINING ACUTE ACCENT */ 0x49, /* LATIN CAPITAL LETTER I */ 0xCC, 0x82, /* COMBINING CIRCUMFLEX ACCENT */ 0x49, /* LATIN CAPITAL LETTER I */ 0xCC, 0x88, /* COMBINING DIAERESIS */ 0x4E, /* LATIN CAPITAL LETTER N */ 0xCC, 0x83, /* COMBINING TILDE */ 0x00 }; /* NFC normalized UTF-8 encoding for same Unicode string: */ unsigned char nname_utf8[] = { 0xC3, 0x80, /* LATIN CAPITAL LETTER A WITH GRAVE */ 0xC3, 0x81, /* LATIN CAPITAL LETTER A WITH ACUTE */ 0xC3, 0x82, /* LATIN CAPITAL LETTER A WITH CIRCUMFLEX */ 0xC3, 0x83, /* LATIN CAPITAL LETTER A WITH TILDE */ 0xC3, 0x84, /* LATIN CAPITAL LETTER A WITH DIAERESIS */ 0xC3, 0x85, /* LATIN CAPITAL LETTER A WITH RING ABOVE */ 0xC3, 0x87, /* LATIN CAPITAL LETTER C WITH CEDILLA */ 0xC3, 0x88, /* LATIN CAPITAL LETTER E WITH GRAVE */ 0xC3, 0x89, /* LATIN CAPITAL LETTER E WITH ACUTE */ 0xC3, 0x8A, /* LATIN CAPITAL LETTER E WITH CIRCUMFLEX */ 0xC3, 0x8B, /* LATIN CAPITAL LETTER E WITH DIAERESIS */ 0xC3, 0x8C, /* LATIN CAPITAL LETTER I WITH GRAVE */ 0xC3, 0x8D, /* LATIN CAPITAL LETTER I WITH ACUTE */ 0xC3, 0x8E, /* LATIN CAPITAL LETTER I WITH CIRCUMFLEX */ 0xC3, 0x8F, /* LATIN CAPITAL LETTER I WITH DIAERESIS */ 0xC3, 0x91, /* LATIN CAPITAL LETTER N WITH TILDE */ 0x00 }; /* Unnormalized name used for dimension, variable, and attribute value */ #define UNAME ((char *) uname_utf8) #define UNAMELEN (sizeof uname_utf8) /* Normalized name */ #define NNAME ((char *) nname_utf8) #define NNAMELEN (sizeof nname_utf8) char name_in[UNAMELEN + 1], strings_in[UNAMELEN + 1]; nc_type att_type; MPI_Offset att_len; int err, dimid_in, varid_in, attnum_in; int attvals[] = {42}; #define ATTNUM ((sizeof attvals)/(sizeof attvals[0])) err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL,&ncid); ERR /* Define dimension with unnormalized Unicode UTF-8 encoded name */ err = ncmpi_def_dim(ncid, UNAME, NX, &dimid); ERR dimids[0] = dimid; /* Define variable with same name */ err = ncmpi_def_var(ncid, UNAME, NC_CHAR, NDIMS, dimids, &varid); ERR /* Create string attribute with same value */ err = ncmpi_put_att_text(ncid, varid, UNITS, UNAMELEN, UNAME); ERR /* Create int attribute with same name */ err = ncmpi_put_att_int(ncid, varid, UNAME, NC_INT, ATTNUM, attvals); ERR /* Try to create dimension and variable with NFC-normalized * version of same name. These should fail, as unnormalized name * should have been normalized in library, so these are attempts to * create duplicate netCDF objects. */ if ((err = ncmpi_def_dim(ncid, NNAME, NX, &dimid)) != NC_ENAMEINUSE) { printf("Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err); return 1; } if ((err=ncmpi_def_var(ncid, NNAME, NC_CHAR, NDIMS, dimids, &varid)) != NC_ENAMEINUSE) { printf("Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err); return 1; } err = ncmpi_enddef(ncid); ERR /* Write string data, UTF-8 encoded, to the file */ err = ncmpi_put_var_text_all(ncid, varid, UNAME); ERR err = ncmpi_close(ncid); ERR /* Check it out. */ err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err = ncmpi_inq_varid(ncid, UNAME, &varid); ERR err = ncmpi_inq_varname(ncid, varid, name_in); ERR err = strncmp(NNAME, name_in, NNAMELEN); ERR err = ncmpi_inq_varid(ncid, NNAME, &varid_in); ERR if ((err = ncmpi_inq_dimid(ncid, UNAME, &dimid_in)) || dimid != dimid_in) {printf("Error at line %d\n",__LINE__);return 1;} if ((err = ncmpi_inq_dimid(ncid, NNAME, &dimid_in)) || dimid != dimid_in) {printf("Error at line %d\n",__LINE__);return 1;} err = ncmpi_inq_att(ncid, varid, UNITS, &att_type, &att_len); ERR if ( att_type != NC_CHAR || att_len != UNAMELEN) {printf("Error at line %d\n",__LINE__);return 1;} err = ncmpi_get_att_text(ncid, varid, UNITS, strings_in); ERR strings_in[UNAMELEN] = '\0'; err = strncmp(UNAME, strings_in, UNAMELEN); ERR if ((err = ncmpi_inq_attid(ncid, varid, UNAME, &attnum_in)) || ATTNUM != attnum_in) {printf("Error at line %d\n",__LINE__);return 1;} if ((err = ncmpi_inq_attid(ncid, varid, NNAME, &attnum_in)) || ATTNUM != attnum_in) {printf("Error at line %d\n",__LINE__);return 1;} err = ncmpi_close(ncid); ERR return 0; }
int main(int argc, char **argv) { char dir_name[256], filename[256]; int err, rank, nerrs=0, format, ncid; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc != 2) { if (!rank) printf("Usage: %s dir_name\n",argv[0]); MPI_Finalize(); return 0; } strcpy(dir_name, argv[1]); MPI_Bcast(dir_name, 256, MPI_CHAR, 0, MPI_COMM_WORLD); if (rank == 0) { char cmd_str[256]; sprintf(cmd_str, "*** TESTING C %s for inquiring CDF file formats ", argv[0]); printf("%-66s ------ ", cmd_str); } /* test CDF-1 -----------------------------------------------------------*/ sprintf(filename,"%s/test_cdf1.nc",dir_name); err = ncmpi_open(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid); ERR err = ncmpi_inq_format(ncid, &format); ERR if (format != NC_FORMAT_CLASSIC) { printf("Error (line=%d): expecting CDF-1 format for file %s but got %d\n", __LINE__,filename,format); nerrs++; } err = ncmpi_close(ncid); ERR err = ncmpi_inq_file_format(filename, &format); ERR if (format != NC_FORMAT_CLASSIC) { printf("Error (line=%d): expecting CDF-1 format for file %s but got %d\n", __LINE__,filename,format); nerrs++; } /* test CDF-2 -----------------------------------------------------------*/ sprintf(filename,"%s/test_cdf2.nc",dir_name); err = ncmpi_open(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid); ERR err = ncmpi_inq_format(ncid, &format); ERR if (format != NC_FORMAT_CDF2) { printf("Error (line=%d): expecting CDF-2 format for file %s but got %d\n", __LINE__,filename,format); nerrs++; } err = ncmpi_close(ncid); ERR err = ncmpi_inq_file_format(filename, &format); ERR if (format != NC_FORMAT_CDF2) { printf("Error (line=%d): expecting CDF-2 format for file %s but got %d\n", __LINE__,filename,format); nerrs++; } /* test CDF-5 -----------------------------------------------------------*/ sprintf(filename,"%s/test_cdf5.nc",dir_name); err = ncmpi_open(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid); ERR err = ncmpi_inq_format(ncid, &format); ERR if (format != NC_FORMAT_CDF5) { printf("Error (line=%d): expecting CDF-5 format for file %s but got %d\n", __LINE__,filename,format); nerrs++; } err = ncmpi_close(ncid); ERR err = ncmpi_inq_file_format(filename, &format); ERR if (format != NC_FORMAT_CDF5) { printf("Error (line=%d): expecting CDF-5 format for file %s but got %d\n", __LINE__,filename,format); nerrs++; } MPI_Offset malloc_size, sum_size; err = ncmpi_inq_malloc_size(&malloc_size); if (err == NC_NOERR) { MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && sum_size > 0) printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", sum_size); } MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); else printf(PASS_STR); } MPI_Finalize(); return 0; }
/*----< main() >------------------------------------------------------------*/ int main(int argc, char **argv) { int i, j, err, nerrs=0, rank, nprocs; int ncid, dimid[2], varid, req, status; MPI_Offset start[2], count[2], stride[2], imap[2]; int var[6][4]; float k, rh[4][6]; signed char varT[4][6]; char filename[256]; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (argc > 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); MPI_Finalize(); return 1; } if (argc == 2) snprintf(filename, 256, "%s", argv[1]); else strcpy(filename, "testfile.nc"); if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); sprintf(cmd_str, "*** TESTING C %s for get/put varm ", basename(argv[0])); printf("%-66s ------ ", cmd_str); fflush(stdout); free(cmd_str); } #ifdef DEBUG if (nprocs > 1 && rank == 0) printf("Warning: %s is designed to run on 1 process\n", argv[0]); #endif err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_DATA, MPI_INFO_NULL, &ncid); CHECK_ERR /* define a variable of a 6 x 4 integer array in the nc file */ err = ncmpi_def_dim(ncid, "Y", 6, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 4, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR /* create a 6 x 4 integer variable in the file with contents: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 */ for (j=0; j<6; j++) for (i=0; i<4; i++) var[j][i] = j*4+i; start[0] = 0; start[1] = 0; count[0] = 6; count[1] = 4; if (rank > 0) count[0] = count[1] = 0; err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR if (nprocs > 1) MPI_Barrier(MPI_COMM_WORLD); err = ncmpi_close(ncid); CHECK_ERR err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR /* read the variable back in the matrix transposed way, rh is 4 x 6 */ count[0] = 6; count[1] = 4; stride[0] = 1; stride[1] = 1; imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */ for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; err = ncmpi_iget_varm_float(ncid, varid, start, count, stride, imap, &rh[0][0], &req); CHECK_ERR err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR err = status; CHECK_ERR /* check the contents of read */ k = 0.0; for (i=0; i<6; i++) { for (j=0; j<4; j++) { if (rh[j][i] != k) { #ifdef PRINT_ERR_ON_SCREEN printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n", __LINE__,__FILE__,j,i,k,rh[j][i]); #endif nerrs++; break; } k += 1.0; } } #ifdef PRINT_ON_SCREEN /* print the contents of read */ for (j=0; j<4; j++) { printf("[%2d]: ",j); for (i=0; i<6; i++) { printf("%5.1f",rh[j][i]); } printf("\n"); } #endif /* the stdout should be: [ 0]: 0.0 4.0 8.0 12.0 16.0 20.0 [ 1]: 1.0 5.0 9.0 13.0 17.0 21.0 [ 2]: 2.0 6.0 10.0 14.0 18.0 22.0 [ 3]: 3.0 7.0 11.0 15.0 19.0 23.0 */ for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, &rh[0][0]); CHECK_ERR /* check the contents of read */ k = 0.0; for (i=0; i<6; i++) { for (j=0; j<4; j++) { if (rh[j][i] != k) { #ifdef PRINT_ERR_ON_SCREEN printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n", __LINE__,__FILE__,j,i,k,rh[j][i]); #endif nerrs++; break; } k += 1.0; } } #ifdef PRINT_ON_SCREEN /* print the contents of read */ for (j=0; j<4; j++) { printf("[%2d]: ",j); for (i=0; i<6; i++) { printf("%5.1f",rh[j][i]); } printf("\n"); } #endif /* the stdout should be: [ 0]: 0.0 4.0 8.0 12.0 16.0 20.0 [ 1]: 1.0 5.0 9.0 13.0 17.0 21.0 [ 2]: 2.0 6.0 10.0 14.0 18.0 22.0 [ 3]: 3.0 7.0 11.0 15.0 19.0 23.0 */ err = ncmpi_close(ncid); CHECK_ERR err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR /* testing get_varm(), first zero-out the variable in the file */ memset(&var[0][0], 0, 6*4*sizeof(int)); start[0] = 0; start[1] = 0; count[0] = 6; count[1] = 4; if (rank > 0) count[0] = count[1] = 0; err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR /* set the contents of the write buffer varT, a 4 x 6 char array 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73 */ for (j=0; j<4; j++) for (i=0; i<6; i++) varT[j][i] = j*6+i + 50; /* write varT to the NC variable in the matrix transposed way */ start[0] = 0; start[1] = 0; count[0] = 6; count[1] = 4; stride[0] = 1; stride[1] = 1; imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */ if (rank > 0) count[0] = count[1] = 0; err = ncmpi_iput_varm_schar(ncid, varid, start, count, stride, imap, &varT[0][0], &req); CHECK_ERR err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR err = status; CHECK_ERR /* the output from command "ncmpidump -v var test.nc" should be: var = 50, 56, 62, 68, 51, 57, 63, 69, 52, 58, 64, 70, 53, 59, 65, 71, 54, 60, 66, 72, 55, 61, 67, 73 ; */ /* check if the contents of write buffer have been altered */ for (j=0; j<4; j++) { for (i=0; i<6; i++) { if (varT[j][i] != j*6+i + 50) { #ifdef PRINT_ERR_ON_SCREEN /* this error is a pnetcdf internal error, if occurs */ printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n", __LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]); #endif nerrs++; break; } } } err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, imap, &varT[0][0]); CHECK_ERR /* check if the contents of write buffer have been altered */ for (j=0; j<4; j++) { for (i=0; i<6; i++) { if (varT[j][i] != j*6+i + 50) { #ifdef PRINT_ERR_ON_SCREEN /* this error is a pnetcdf internal error, if occurs */ printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n", __LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]); #endif nerrs++; break; } } } err = ncmpi_close(ncid); CHECK_ERR /* check if PnetCDF freed all internal malloc */ MPI_Offset malloc_size, sum_size; err = ncmpi_inq_malloc_size(&malloc_size); if (err == NC_NOERR) { MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && sum_size > 0) printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", sum_size); if (malloc_size > 0) ncmpi_inq_malloc_list(); } MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); else printf(PASS_STR); } MPI_Finalize(); return (nerrs > 0); }
/* * NETCDF_open() * Open the trajectory specified by the filename and accessMode in trajInfo as * a NETCDF traj. * Return 0 on success, 1 on failure */ int NETCDF_open(coordinateInfo *trajInfo) { #ifdef BINTRAJ int err,ncid; // NOTE: Put in a check, only open if coord is unknown or netcdf if (prnlev>0) fprintf(stdout,"[%i] NETCDF_open(): Opening %s\n", worldrank,trajInfo->filename); switch (trajInfo->accessMode) { case 0: // Read # ifdef MPI err = ncmpi_open(MPI_COMM_WORLD, trajInfo->filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); /* This next line is a test. Apparently it puts the netcdf file in an * independent I/O mode. Not sure if it is bad to always put here. * Originally this call was only made from ptrajPreprocess... */ if (err == NC_NOERR) err = ncmpi_begin_indep_data(ncid); # else err = nc_open(trajInfo->filename, NC_NOWRITE, &ncid); # endif break; case 1: // Write //omode=NC_WRITE; # ifdef MPI err = ncmpi_create(MPI_COMM_WORLD, trajInfo->filename, NC_64BIT_OFFSET, MPI_INFO_NULL, &ncid); if (err == NC_NOERR) ncmpi_begin_indep_data(ncid); # else err = nc_create(trajInfo->filename, NC_64BIT_OFFSET, &ncid); # endif break; case 2: // Append printfone("Appending of NETCDF files is not supported.\n"); return 1; break; } /* If opening succeeded and memory hasnt been allocated already * initialize necessary data structure. * NOTE: Should this be in NETCDF_setup? If so ncid would have to * be its own variable in coordinateInfo. * NOTE: If this is an output file trajInfo->type has already been set. * Not a huge problem but is a bit circular. TRAJOUT should eventually * only set trajInfo->isNetcdf. */ if (err == NC_NOERR) { trajInfo->type = COORD_AMBER_NETCDF; if (trajInfo->NCInfo==NULL) { trajInfo->NCInfo = (netcdfTrajectoryInfo *) safe_malloc(sizeof(netcdfTrajectoryInfo)); INITIALIZE_netcdfTrajectoryInfo( trajInfo->NCInfo ); } trajInfo->NCInfo->currentFrame = worldrank; // Always set NCID since it can change depending on when file is opened trajInfo->NCInfo->ncid = ncid; if (prnlev>0) fprintf(stdout,"NETCDF_open(): %s has been assigned ncid of %i\n", trajInfo->filename,ncid); return 0; } // If we are here an error occured. Print the error message before exiting. fprintf(stdout,"Error: NETCDF_open(): Could not open %s with accessMode %i\n", trajInfo->filename,trajInfo->accessMode); fprintf(stdout,"%s\n",nc_strerror(err)); #endif // If no BINTRAJ always fail return 1; }
int main(int argc, char **argv) { int stat; /* return status */ int ncid; /* netCDF id */ int rec, i, j, k; signed char x[] = {42, 21}; /* dimension ids */ int rec_dim; int i_dim; int j_dim; int k_dim; int n_dim; #define NUMRECS 1 #define I_LEN 4104 #define J_LEN 1023 #define K_LEN 1023 #define N_LEN 2 /* dimension lengths */ MPI_Offset rec_len = NC_UNLIMITED; MPI_Offset i_len = I_LEN; MPI_Offset j_len = J_LEN; MPI_Offset k_len = K_LEN; MPI_Offset n_len = N_LEN; /* variable ids */ int var1_id; int x_id; /* rank (number of dimensions) for each variable */ # define RANK_var1 4 # define RANK_x 2 /* variable shapes */ int var1_dims[RANK_var1]; int x_dims[RANK_x]; printf("\n*** Testing large files, slowly.\n"); printf("*** Creating large file %s...", FILE_NAME); MPI_Init(&argc, &argv); /* enter define mode */ stat = ncmpi_create(MPI_COMM_WORLD, FILE_NAME, NC_CLOBBER|NC_64BIT_OFFSET, MPI_INFO_NULL, &ncid); check_err(stat,__LINE__,__FILE__); /* define dimensions */ stat = ncmpi_def_dim(ncid, "rec", rec_len, &rec_dim); check_err(stat,__LINE__,__FILE__); stat = ncmpi_def_dim(ncid, "i", i_len, &i_dim); check_err(stat,__LINE__,__FILE__); stat = ncmpi_def_dim(ncid, "j", j_len, &j_dim); check_err(stat,__LINE__,__FILE__); stat = ncmpi_def_dim(ncid, "k", k_len, &k_dim); check_err(stat,__LINE__,__FILE__); stat = ncmpi_def_dim(ncid, "n", n_len, &n_dim); check_err(stat,__LINE__,__FILE__); /* define variables */ var1_dims[0] = rec_dim; var1_dims[1] = i_dim; var1_dims[2] = j_dim; var1_dims[3] = k_dim; stat = ncmpi_def_var(ncid, "var1", NC_BYTE, RANK_var1, var1_dims, &var1_id); check_err(stat,__LINE__,__FILE__); x_dims[0] = rec_dim; x_dims[1] = n_dim; stat = ncmpi_def_var(ncid, "x", NC_BYTE, RANK_x, x_dims, &x_id); check_err(stat,__LINE__,__FILE__); /* don't initialize variables with fill values */ stat = ncmpi_set_fill(ncid, NC_NOFILL, 0); check_err(stat,__LINE__,__FILE__); /* leave define mode */ stat = ncmpi_enddef (ncid); check_err(stat,__LINE__,__FILE__); { /* store var1 */ int n = 0; static signed char var1[J_LEN][K_LEN]; static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0}; static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN}; static MPI_Offset x_start[RANK_x] = {0, 0}; static MPI_Offset x_count[RANK_x] = {1, N_LEN}; for(rec=0; rec<NUMRECS; rec++) { var1_start[0] = rec; x_start[0] = rec; for(i=0; i<I_LEN; i++) { for(j=0; j<J_LEN; j++) { for (k=0; k<K_LEN; k++) { var1[j][k] = n++; } } var1_start[1] = i; stat = ncmpi_put_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]); check_err(stat,__LINE__,__FILE__); } } stat = ncmpi_put_vara_schar_all(ncid, x_id, x_start, x_count, x); check_err(stat,__LINE__,__FILE__); } stat = ncmpi_close(ncid); check_err(stat,__LINE__,__FILE__); printf("ok\n"); printf("*** Reading large file %s...", FILE_NAME); stat = ncmpi_open(MPI_COMM_WORLD, FILE_NAME, NC_NOWRITE, MPI_INFO_NULL, &ncid); check_err(stat,__LINE__,__FILE__); { /* read var1 */ int n = 0; static signed char var1[J_LEN][K_LEN]; static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0}; static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN}; static MPI_Offset x_start[RANK_x] = {0, 0}; static MPI_Offset x_count[RANK_x] = {1, N_LEN}; for(rec=0; rec<NUMRECS; rec++) { var1_start[0] = rec; x_start[0] = rec; for(i=0; i<I_LEN; i++) { var1_start[1] = i; stat = ncmpi_get_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]); check_err(stat,__LINE__,__FILE__); for(j=0; j<J_LEN; j++) { for (k=0; k<K_LEN; k++) { if (var1[j][k] != (signed char) n) { printf("Error on read, var1[%d, %d, %d, %d] = %d wrong, " "should be %d !\n", rec, i, j, k, var1[j][k], (signed char) n); return 1; } n++; } } } ncmpi_get_vara_schar_all(ncid, x_id, x_start, x_count, x); if(x[0] != 42 || x[1] != 21) { printf("Error on read, x[] = %d, %d\n", x[0], x[1]); return 1; } } } stat = ncmpi_close(ncid); check_err(stat,__LINE__,__FILE__); printf("ok\n"); printf("*** Tests successful!\n"); /* Delete the file. */ (void) remove(FILE_NAME); MPI_Finalize(); return 0; }
void read_pnetcdf(const std::string &filename, int iVar, ConfigMap &configMap, HostArray<double> &localData) { int myRank; MPI_Comm_rank(MPI_COMM_WORLD, &myRank); int nbMpiProc; MPI_Comm_size(MPI_COMM_WORLD, &nbMpiProc); // netcdf file id int ncFileId; int err; // ghostWidth int ghostWidth = configMap.getInteger("mesh","ghostWidth",3); // file creation mode int ncOpenMode = NC_NOWRITE; int varIds[8]; MPI_Offset starts[3], counts[3]; // read_size, sum_read_size; MPI_Info mpi_info_used; // domain local size int nx,ny,nz; // sizes to read //int nx_r, ny_r, nz_r; // logical sizes / per sub-domain //int nx_g, ny_g, nz_g; // sizes with ghost zones included / per sub-domain /* read domain sizes */ nx=configMap.getInteger("mesh","nx",32); ny=configMap.getInteger("mesh","ny",32); nz=configMap.getInteger("mesh","nz",32); // nx_g = nx+2*ghostWidth; // ny_g = ny+2*ghostWidth; // nz_g = nz+2*ghostWidth; // get input filename from configMap //std::string filename = configMap.getString("input", "filename", ""); /* * Open NetCDF file */ err = ncmpi_open(MPI_COMM_WORLD, filename.c_str(), ncOpenMode, MPI_INFO_NULL, &ncFileId); if (err != NC_NOERR) { printf("Error: ncmpi_open() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err)); MPI_Abort(MPI_COMM_WORLD, -1); exit(1); } /* * Query NetCDF mode */ int NC_mode; err = ncmpi_inq_version(ncFileId, &NC_mode); if (myRank==0) { if (NC_mode == NC_64BIT_DATA) std::cout << "Pnetcdf Input mode : NC_64BIT_DATA (CDF-5)\n"; else if (NC_mode == NC_64BIT_OFFSET) std::cout << "Pnetcdf Input mode : NC_64BIT_OFFSET (CDF-2)\n"; else std::cout << "Pnetcdf Input mode : unknown\n"; } /* * Query information about variables */ { int ndims, nvars, ngatts, unlimited; err = ncmpi_inq(ncFileId, &ndims, &nvars, &ngatts, &unlimited); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "rho", &varIds[ID]); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "E", &varIds[IP]); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "rho_vx", &varIds[IU]); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "rho_vy", &varIds[IV]); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "rho_vz", &varIds[IW]); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "Bx", &varIds[IA]); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "By", &varIds[IB]); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "Bz", &varIds[IC]); PNETCDF_HANDLE_ERROR; } // end query information /* * Define expected data types (no conversion done here) */ //nc_type ncDataType; MPI_Datatype mpiDataType; //ncDataType = NC_DOUBLE; mpiDataType = MPI_DOUBLE; /* * Get all the MPI_IO hints used (just in case, we want to print it after * reading data... */ err = ncmpi_get_file_info(ncFileId, &mpi_info_used); PNETCDF_HANDLE_ERROR; /* * Read heavy data (take care of row-major / column major format !) */ // use overlapping domains // counts[IZ] = nx_rg; // counts[IY] = ny_rg; // counts[IX] = nz_rg; // starts[IZ] = 0; // starts[IY] = 0; // starts[IX] = myRank*nz_r; counts[IZ] = nx; counts[IY] = ny; counts[IX] = nz; starts[IZ] = ghostWidth; starts[IY] = ghostWidth; starts[IX] = ghostWidth+myRank*nz; int nItems = counts[IX]*counts[IY]*counts[IZ]; /* * Actual reading */ { double* data; //data = &(localData(0,0,0,0)); data = localData.data(); err = ncmpi_get_vara_all(ncFileId, varIds[iVar], starts, counts, data, nItems, mpiDataType); PNETCDF_HANDLE_ERROR; } // end for loop reading heavy data /* * close the file */ err = ncmpi_close(ncFileId); PNETCDF_HANDLE_ERROR; } // read_pnetcdf
//---------------------------------------------------------------- int main(int argc, char **argv) { init_mpi(argc, argv); printf("start %d %d\n",rank,process_count); int i; if(rank==0) { parse_args(argc, argv); // var_count = read_list_in_file(var_file, &netcdf_var_names); // rank_0_print("Number of variables = %d\n", var_count); // time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names); // rank_0_print("Number of timesteps = %d\n", time_step_count); check_args(); } // The command line arguments are shared by all processes #if PIDX_HAVE_MPI MPI_Bcast(global_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD); MPI_Bcast(local_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD); // MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&var_file, 512, MPI_CHAR, 0, MPI_COMM_WORLD); MPI_Bcast(&netcdf_file_list, 512, MPI_CHAR, 0, MPI_COMM_WORLD); // MPI_Bcast(&var_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&output_file_name, 512, MPI_CHAR, 0, MPI_COMM_WORLD); #endif //TODO: Only the rank 0 should read the input files and broadcast the data var_count = read_list_in_file(var_file, &netcdf_var_names); rank_0_print("Number of variables = %d\n", var_count); time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names); rank_0_print("Number of timesteps = %d\n", time_step_count); calculate_per_process_offsets(); create_pidx_var_names(); PIDX_access pidx_access; create_pidx_access(&pidx_access); PIDX_time_step_caching_ON(); determine_var_types(); create_pidx_vars(); int t = 0,plist_id; for (t = 0; t < time_step_count; ++t) { rank_0_print("Processing time step %d (file %s)\n", t, netcdf_file_names[t]); PIDX_file pidx_file; int ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, pidx_access, &pidx_file); if (ret != PIDX_success) terminate_with_error_msg("ERROR: Failed to create PIDX file\n"); set_pidx_params(pidx_file); PIDX_set_current_time_step(pidx_file, t); int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[t], NC_NOWRITE, MPI_INFO_NULL, &plist_id); if (file_id !=0) terminate_with_error_msg("ERROR: Failed to open file %s\n", netcdf_file_names[t]); int v = 0; for(v = 0; v < var_count; ++v) { rank_0_print("Processing variable %s\n", netcdf_var_names[v]); var_data = malloc(var_types[v].atomic_type * var_types[v].num_values * local_box_size[0] * local_box_size[1] * local_box_size[2]); read_var_from_netcdf(plist_id, netcdf_var_names[v], var_types[v]); write_var_to_idx(pidx_file, pidx_var_names[v], pidx_vars[v]); if (PIDX_flush(pidx_file) != PIDX_success) terminate_with_error_msg("ERROR: Failed to flush variable %s, time step %d\n", pidx_var_names[v], t); free(var_data); } ncmpi_close(plist_id); PIDX_close(pidx_file); } PIDX_time_step_caching_OFF(); PIDX_close_access(pidx_access); free_memory(); shutdown_mpi(); return 0; }
/*----< main() >------------------------------------------------------------*/ int main(int argc, char **argv) { char filename[256]; int i, j, err, ncid, varid0, varid1, varid2, dimids[2], nerrs=0; int rank, nprocs, debug=0, blocklengths[2], **buf, *bufptr; int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; MPI_Offset start[2], count[2]; MPI_Aint a0, a1, disps[2]; MPI_Datatype buftype, ghost_buftype, rec_filetype, fix_filetype; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc > 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); MPI_Finalize(); return 0; } strcpy(filename, "testfile.nc"); if (argc == 2) strcpy(filename, argv[1]); MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); if (rank == 0) { char cmd_str[256]; sprintf(cmd_str, "*** TESTING C %s for flexible put and get ", argv[0]); printf("%-66s ------ ", cmd_str); fflush(stdout); } buf = (int**)malloc(NY * sizeof(int*)); buf[0] = (int*)malloc(NY * NX * sizeof(int)); for (i=1; i<NY; i++) buf[i] = buf[i-1] + NX; /* construct various MPI derived data types */ /* construct an MPI derived data type for swapping 1st row with 2nd row */ blocklengths[0] = blocklengths[1] = NX; MPI_Get_address(buf[1], &a0); MPI_Get_address(buf[0], &a1); disps[0] = 0; disps[1] = a1 - a0; bufptr = buf[1]; err = MPI_Type_create_hindexed(2, blocklengths, disps, MPI_INT, &buftype); if (err != MPI_SUCCESS) printf("MPI error MPI_Type_create_hindexed\n"); MPI_Type_commit(&buftype); start[0] = 0; start[1] = NX*rank; count[0] = 2; count[1] = NX; if (debug) printf("put start=%lld %lld count=%lld %lld\n",start[0],start[1],count[0],count[1]); /* create a file type for the fixed-size variable */ array_of_sizes[0] = 2; array_of_sizes[1] = NX*nprocs; array_of_subsizes[0] = count[0]; array_of_subsizes[1] = count[1]; array_of_starts[0] = start[0]; array_of_starts[1] = start[1]; MPI_Type_create_subarray(2, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_INT, &fix_filetype); MPI_Type_commit(&fix_filetype); /* create a buftype with ghost cells on each side */ array_of_sizes[0] = count[0]+4; array_of_sizes[1] = count[1]+4; array_of_subsizes[0] = count[0]; array_of_subsizes[1] = count[1]; array_of_starts[0] = 2; array_of_starts[1] = 2; MPI_Type_create_subarray(2, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_INT, &ghost_buftype); MPI_Type_commit(&ghost_buftype); /* create a new file for write */ err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); ERR /* define a 2D array */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimids[0]); ERR err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimids[1]); ERR err = ncmpi_def_var(ncid, "rec_var", NC_INT, 2, dimids, &varid0); ERR err = ncmpi_def_var(ncid, "dummy_rec", NC_INT, 2, dimids, &varid2); ERR err = ncmpi_def_dim(ncid, "FIX_DIM", 2, &dimids[0]); ERR err = ncmpi_def_var(ncid, "fix_var", NC_INT, 2, dimids, &varid1); ERR err = ncmpi_enddef(ncid); ERR /* create a file type for the record variable */ int *array_of_blocklengths=(int*) malloc(count[0]*sizeof(int)); MPI_Aint *array_of_displacements=(MPI_Aint*) malloc(count[0]*sizeof(MPI_Aint)); MPI_Offset recsize; err = ncmpi_inq_recsize(ncid, &recsize); for (i=0; i<count[0]; i++) { array_of_blocklengths[i] = count[1]; array_of_displacements[i] = start[1]*sizeof(int) + recsize * i; } MPI_Type_create_hindexed(2, array_of_blocklengths, array_of_displacements, MPI_INT, &rec_filetype); MPI_Type_commit(&rec_filetype); free(array_of_blocklengths); free(array_of_displacements); /* initialize the contents of the array */ for (j=0; j<NY; j++) for (i=0; i<NX; i++) buf[j][i] = rank*100 + j*10 + i; /* write the record variable */ err = ncmpi_put_vard_all(ncid, varid0, rec_filetype, bufptr, 1, buftype); ERR /* check if the contents of buf are altered */ CHECK_VALUE /* check if root process can write to file header in data mode */ err = ncmpi_rename_var(ncid, varid0, "rec_VAR"); ERR /* write the fixed-size variable */ err = ncmpi_put_vard_all(ncid, varid1, fix_filetype, bufptr, 1, buftype); ERR /* check if the contents of buf are altered */ CHECK_VALUE /* check if root process can write to file header in data mode */ err = ncmpi_rename_var(ncid, varid0, "rec_var"); ERR /* test the same routines in independent data mode */ err = ncmpi_begin_indep_data(ncid); ERR err = ncmpi_put_vard(ncid, varid0, rec_filetype, bufptr, 1, buftype); ERR CHECK_VALUE err = ncmpi_rename_var(ncid, varid0, "rec_VAR"); ERR err = ncmpi_put_vard(ncid, varid1, fix_filetype, bufptr, 1, buftype); ERR CHECK_VALUE err = ncmpi_rename_var(ncid, varid0, "rec_var"); ERR err = ncmpi_end_indep_data(ncid); ERR err = ncmpi_close(ncid); ERR /* open the same file and read back for validate */ err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err = ncmpi_inq_varid(ncid, "rec_var", &varid0); ERR err = ncmpi_inq_varid(ncid, "fix_var", &varid1); ERR nerrs += get_var_and_verify(ncid, varid0, start, count, buf, buftype, ghost_buftype, rec_filetype); nerrs += get_var_and_verify(ncid, varid1, start, count, buf, buftype, ghost_buftype, fix_filetype); err = ncmpi_close(ncid); ERR MPI_Type_free(&rec_filetype); MPI_Type_free(&fix_filetype); MPI_Type_free(&buftype); MPI_Type_free(&ghost_buftype); free(buf[0]); free(buf); /* check if PnetCDF freed all internal malloc */ MPI_Offset malloc_size, sum_size; err = ncmpi_inq_malloc_size(&malloc_size); if (err == NC_NOERR) { MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && sum_size > 0) printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", sum_size); } MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); else printf(PASS_STR); } MPI_Finalize(); return 0; }
int main(int argc, char **argv) { int rank, nprocs, ncid, pres_varid, temp_varid; int lat_varid, lon_varid; /* The start and count arrays will tell the netCDF library where to read our data. */ MPI_Offset start[NDIMS], count[NDIMS]; /* Program variables to hold the data we will read. We will only need enough space to hold one timestep of data; one record. */ float **pres_in=NULL; /* [NLVL/nprocs][NLAT][NLON] */ float **temp_in=NULL; /* [NLVL/nprocs][NLAT][NLON] */ /* These program variables hold the latitudes and longitudes. */ float lats[NLAT], lons[NLON]; /* Loop indexes. */ int lvl, lat, lon, rec, i = 0; /* Error handling. */ int err, nerrs=0; char *filename = FILE_NAME; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc > 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); MPI_Finalize(); return 1; } if (argc == 2) filename = argv[1]; if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); sprintf(cmd_str, "*** TESTING C %s for reading file", basename(argv[0])); printf("%-66s ------ ", cmd_str); free(cmd_str); } /* Open the file. */ err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR /* Get the varids of the latitude and longitude coordinate * variables. */ err = ncmpi_inq_varid(ncid, LAT_NAME, &lat_varid); CHECK_ERR err = ncmpi_inq_varid(ncid, LON_NAME, &lon_varid); CHECK_ERR /* Read the coordinate variable data. */ memset(lats, 0, sizeof(float)*NLAT); memset(lons, 0, sizeof(float)*NLON); err = ncmpi_get_var_float_all(ncid, lat_varid, &lats[0]); CHECK_ERR err = ncmpi_get_var_float_all(ncid, lon_varid, &lons[0]); CHECK_ERR /* Check the coordinate variable data. */ for (lat = 0; lat < NLAT; lat++) if (lats[lat] != START_LAT + 5.*lat) { printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LAT+5.*lat,lats[lat]); nerrs++; goto fn_exit; } for (lon = 0; lon < NLON; lon++) if (lons[lon] != START_LON + 5.*lon) { printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LON+5.*lon,lons[lon]); nerrs++; goto fn_exit; } /* Get the varids of the pressure and temperature netCDF * variables. */ err = ncmpi_inq_varid(ncid, PRES_NAME, &pres_varid); CHECK_ERR err = ncmpi_inq_varid(ncid, TEMP_NAME, &temp_varid); CHECK_ERR /* Read the data. Since we know the contents of the file we know * that the data arrays in this program are the correct size to * hold one timestep. */ count[0] = 1; count[2] = NLAT; count[3] = NLON; start[2] = 0; start[3] = 0; /* divide NLVL dimension among processes */ count[1] = NLVL / nprocs; start[1] = count[1] * rank; if (rank < NLVL % nprocs) { start[1] += rank; count[1]++; } else { start[1] += NLVL % nprocs; } if (count[1] == 0) start[1] = 0; /* allocate read buffers */ pres_in = (float**) malloc(count[1]*2 * sizeof(float*)); temp_in = pres_in + count[1]; if (count[1] > 0) { pres_in[0] = (float*) malloc(count[1]*2 * NLAT*NLON * sizeof(float)); temp_in[0] = pres_in[0] + count[1] * NLAT*NLON; for (i=1; i<count[1]; i++) { pres_in[i] = pres_in[i-1] + NLAT*NLON; temp_in[i] = temp_in[i-1] + NLAT*NLON; } } /* Read and check one record at a time. */ for (rec = 0; rec < NREC; rec++) { start[0] = rec; err = ncmpi_get_vara_float_all(ncid, pres_varid, start, count, &pres_in[0][0]); CHECK_ERR err = ncmpi_get_vara_float_all(ncid, temp_varid, start, count, &temp_in[0][0]); CHECK_ERR /* Check the data. */ i = (int)start[1] * NLAT * NLON; for (lvl=0; lvl<count[1]; lvl++) for (lat = 0; lat < NLAT; lat++) for (lon = 0; lon < NLON; lon++) { if (pres_in[lvl][lat*NLON+lon] != SAMPLE_PRESSURE + i) { printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_PRESSURE+i,pres_in[lvl][lat*NLON+lon]); nerrs++; goto fn_exit; } if (temp_in[lvl][lat*NLON+lon] != SAMPLE_TEMP + i) { printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_TEMP+i,temp_in[lvl][lat*NLON+lon]); nerrs++; goto fn_exit; } i++; } } /* next record */ fn_exit: /* Close the file. */ err = ncmpi_close(ncid); CHECK_ERR if (pres_in != NULL) { if (pres_in[0] != NULL) free(pres_in[0]); free(pres_in); } /* check if there is any malloc residue */ MPI_Offset malloc_size, sum_size; err = ncmpi_inq_malloc_size(&malloc_size); if (err == NC_NOERR) { MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && sum_size > 0) printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", sum_size); } MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); else printf(PASS_STR); } MPI_Finalize(); return (nerrs > 0); }
int main(int argc, char **argv) { int i, j, rank, nprocs, ret; int ncfile, ndims, nvars, ngatts, unlimited; int var_ndims, var_natts;; MPI_Offset *dim_sizes, var_size; MPI_Offset *start, *count; int *requests, *statuses; char varname[NC_MAX_NAME+1]; int dimids[NC_MAX_VAR_DIMS]; nc_type type; int **data; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (argc != 2) { if (rank == 0) printf("Usage: %s filename\n", argv[0]); MPI_Finalize(); exit(-1); } ret = ncmpi_open(MPI_COMM_WORLD, argv[1], NC_NOWRITE, MPI_INFO_NULL, &ncfile); if (ret != NC_NOERR) handle_error(ret, __LINE__); /* reader knows nothing about dataset, but we can interrogate with query * routines: ncmpi_inq tells us how many of each kind of "thing" * (dimension, variable, attribute) we will find in the file */ /* no commnunication needed after ncmpi_open: all processors have a cached * veiw of the metadata once ncmpi_open returns */ ret = ncmpi_inq(ncfile, &ndims, &nvars, &ngatts, &unlimited); if (ret != NC_NOERR) handle_error(ret, __LINE__); /* we do not really need the name of the dimension or the variable for * reading in this example. we could, in a different example, take the * name of a variable on the command line and read just that one */ dim_sizes = calloc(ndims, sizeof(MPI_Offset)); /* netcdf dimension identifiers are allocated sequentially starting * at zero; same for variable identifiers */ for(i=0; i<ndims; i++) { ret = ncmpi_inq_dimlen(ncfile, i, &(dim_sizes[i]) ); if (ret != NC_NOERR) handle_error(ret, __LINE__); } requests = calloc(nvars, sizeof(int)); statuses = calloc(nvars, sizeof(int)); data = malloc(nvars*sizeof(int*)); for(i=0; i<nvars; i++) { /* much less coordination in this case compared to rank 0 doing all * the i/o: everyone already has the necessary information */ ret = ncmpi_inq_var(ncfile, i, varname, &type, &var_ndims, dimids, &var_natts); if (ret != NC_NOERR) handle_error(ret, __LINE__); start = calloc(var_ndims, sizeof(MPI_Offset)); count = calloc(var_ndims, sizeof(MPI_Offset)); /* we will simply decompose along one dimension. Generally the * application has some algorithim for domain decomposistion. Note * that data decomposistion can have an impact on i/o performance. * Often it's best just to do what is natural for the application, * but something to consider if performance is not what was * expected/desired */ start[0] = (dim_sizes[dimids[0]]/nprocs)*rank; count[0] = (dim_sizes[dimids[0]]/nprocs); var_size = count[0]; for (j=1; j<var_ndims; j++) { start[j] = 0; count[j] = dim_sizes[dimids[j]]; var_size *= count[j]; } switch(type) { case NC_INT: data[i] = calloc(var_size, sizeof(int)); /* as with the writes, this call is independent: we * will do any coordination (if desired) in a * subsequent ncmpi_wait_all() call */ ret = ncmpi_iget_vara(ncfile, i, start, count, data[i], var_size, MPI_INT, &(requests[i])); if (ret != NC_NOERR) handle_error(ret, __LINE__); break; default: /* we can do this for all the known netcdf types but this * example is already getting too long */ fprintf(stderr, "unsupported NetCDF type \n"); } free(start); free(count); } ret = ncmpi_wait_all(ncfile, nvars, requests, statuses); if (ret != NC_NOERR) handle_error(ret, __LINE__); /* now that the ncmpi_wait_all has returned, the caller can do stuff with * the buffers passed in to the non-blocking operations. The buffer resue * rules are similar to MPI non-blocking messages */ for (i=0; i<nvars; i++) { if (data[i] != NULL) free(data[i]); } free(data); free(dim_sizes); free(requests); free(statuses); ret = ncmpi_close(ncfile); if (ret != NC_NOERR) handle_error(ret, __LINE__); MPI_Finalize(); return 0; }
/* reads input in pnetcdf format nblocks: (output) local number of blocks tot_blocks: (output) total number of blocks vblocks: (output) pointer to array of vblocks in_file: input file name comm: MPI communicator gids: (output) gids of local blocks (allocated by this function) num_neighbors: (output) number of neighbors for each local block (allocated by this function) neighbors: (output) gids of neighbors of each local block (allocated by this function) side effects: allocates vblocks, gids, num_neighbors, neighbors */ void pnetcdf_read(int *nblocks, int *tot_blocks, struct vblock_t ***vblocks, char *in_file, MPI_Comm comm, int *gids, int *num_neighbors, int **neighbors) { #ifdef USEPNETCDF int err; int ncid, varids[23], dimids[8]; MPI_Offset start[2], count[2]; nc_type type; int ndims, natts; int dims[2]; int rank, groupsize; /* MPI usual */ /* open file for reading */ err = ncmpi_open(comm, in_file, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR; err = ncmpi_inq_varid(ncid, "block_off_num_verts", &varids[5]); ERR; err = ncmpi_inq_varid(ncid, "block_off_num_complete_cells", &varids[6]); ERR; err = ncmpi_inq_varid(ncid, "block_off_tot_num_cell_faces", &varids[7]); ERR; err = ncmpi_inq_varid(ncid, "block_off_tot_num_face_verts", &varids[8]); ERR; err = ncmpi_inq_varid(ncid, "block_off_num_orig_particles", &varids[9]); ERR; /* get number of blocks */ MPI_Offset num_g_blocks; /* 64 bit version of tot_blcoks */ err = ncmpi_inq_dimlen(ncid, dimids[0], &num_g_blocks); ERR; *tot_blocks = num_g_blocks; MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &groupsize); int start_block_ofst = rank * (*tot_blocks / groupsize); *nblocks = (rank < groupsize - 1 ? (*tot_blocks / groupsize) : *tot_blocks - (rank * *tot_blocks / groupsize)); /* block offsets */ int64_t *block_ofsts = (int64_t*)malloc(*tot_blocks * sizeof(int64_t)); *vblocks = (struct vblock_t**)malloc(*nblocks * sizeof(struct vblock_t*)); /* read all blocks */ gids = (int *)malloc(*nblocks * sizeof(int)); num_neighbors = (int *)malloc(*nblocks * sizeof(int)); neighbors = (int **)malloc(*nblocks * sizeof(int *)); int b; for (b = 0; b < *nblocks; b++) { struct vblock_t* v = (struct vblock_t*)malloc(sizeof(struct vblock_t)); /* quantities */ start[0] = start_block_ofst + b; count[0] = 1; err = ncmpi_inq_varid(ncid, "num_verts", &varids[0]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[0], start, count, &(v->num_verts)); ERR; err = ncmpi_inq_varid(ncid, "num_complete_cells", &varids[1]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[1], start, count, &(v->num_complete_cells)); ERR; err = ncmpi_inq_varid(ncid, "tot_num_cell_faces", &varids[2]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[2], start, count, &(v->tot_num_cell_faces)); ERR; err = ncmpi_inq_varid(ncid, "tot_num_face_verts", &varids[3]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[3], start, count, &(v->tot_num_face_verts)); ERR; err = ncmpi_inq_varid(ncid, "num_orig_particles", &varids[4]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[4], start, count, &(v->num_orig_particles)); ERR; err = ncmpi_inq_varid(ncid, "neighbors", &varids[21]); ERR; err = ncmpi_inq_var(ncid, varids[21], 0, &type, &ndims, dimids, &natts); /* block bounds */ start[0] = start_block_ofst + b; start[1] = 0; count[0] = 1; count[1] = 3; err = ncmpi_inq_varid(ncid, "mins", &varids[11]); ERR; err = ncmpi_get_vara_float_all(ncid, varids[11], start, count, v->mins); ERR; err = ncmpi_inq_varid(ncid, "maxs", &varids[12]); ERR; err = ncmpi_get_vara_float_all(ncid, varids[12], start, count, v->maxs); ERR; /* save_verts */ start[0] = 0; count[0] = *tot_blocks; err = ncmpi_get_vara_longlong_all(ncid, varids[5], start, count, block_ofsts); ERR; v->save_verts = (float *)malloc(v->num_verts * 3 * sizeof(float)); start[0] = block_ofsts[start_block_ofst + b]; start[1] = 0; count[0] = v->num_verts; count[1] = 3; err = ncmpi_inq_varid(ncid, "save_verts", &varids[13]); ERR; err = ncmpi_get_vara_float_all(ncid, varids[13], start, count, v->save_verts); ERR; /* sites */ start[0] = 0; count[0] = *tot_blocks; err = ncmpi_get_vara_longlong_all(ncid, varids[9], start, count, block_ofsts); ERR; v->sites = (float *)malloc(v->num_orig_particles * 3 * sizeof(float)); start[0] = block_ofsts[start_block_ofst + b]; start[1] = 0; count[0] = v->num_orig_particles; count[1] = 3; err = ncmpi_inq_varid(ncid, "sites", &varids[14]); ERR; err = ncmpi_get_vara_float_all(ncid, varids[14], start, count, v->sites); ERR; /* complete cells */ start[0] = 0; count[0] = *tot_blocks; err = ncmpi_get_vara_longlong_all(ncid, varids[6], start, count, block_ofsts); ERR; v->complete_cells = (int *)malloc(v->num_complete_cells * sizeof(int)); start[0] = block_ofsts[start_block_ofst + b]; count[0] = v->num_complete_cells; err = ncmpi_inq_varid(ncid, "complete_cells", &varids[15]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[15], start, count, v->complete_cells); ERR; /* areas, uses same block offsets as complete cells */ v->areas = (float *)malloc(v->num_complete_cells * sizeof(float)); start[0] = block_ofsts[start_block_ofst + b]; count[0] = v->num_complete_cells; err = ncmpi_inq_varid(ncid, "areas", &varids[16]); ERR; err = ncmpi_get_vara_float_all(ncid, varids[16], start, count, v->areas); ERR; /* volumes, uses same block offsets as complete cells */ v->vols = (float *)malloc(v->num_complete_cells * sizeof(float)); start[0] = block_ofsts[start_block_ofst + b]; count[0] = v->num_complete_cells; err = ncmpi_inq_varid(ncid, "vols", &varids[17]); ERR; err = ncmpi_get_vara_float_all(ncid, varids[17], start, count, v->vols); ERR; /* num_cell_faces, uses same block offsets as complete cells */ v->num_cell_faces = (int *)malloc(v->num_complete_cells * sizeof(int)); start[0] = block_ofsts[start_block_ofst + b]; count[0] = v->num_complete_cells; err = ncmpi_inq_varid(ncid, "num_cell_faces", &varids[18]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[18], start, count, v->num_cell_faces); ERR; /* num_face_verts */ start[0] = 0; count[0] = *tot_blocks; err = ncmpi_get_vara_longlong_all(ncid, varids[7], start, count, block_ofsts); ERR; v->num_face_verts = (int *)malloc(v->tot_num_cell_faces * sizeof(int)); start[0] = block_ofsts[start_block_ofst + b]; count[0] = v->tot_num_cell_faces; err = ncmpi_inq_varid(ncid, "num_face_verts", &varids[19]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[19], start, count, v->num_face_verts); ERR; /* face_verts */ start[0] = 0; count[0] = *tot_blocks; err = ncmpi_get_vara_longlong_all(ncid, varids[8], start, count, block_ofsts); ERR; v->face_verts = (int *)malloc(v->tot_num_face_verts * sizeof(int)); start[0] = block_ofsts[start_block_ofst + b]; count[0] = v->tot_num_face_verts; err = ncmpi_inq_varid(ncid, "face_verts", &varids[20]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[20], start, count, v->face_verts); ERR; /* neighbors */ MPI_Offset n; /* temporary 64-bit version of number of neighbors */ err = ncmpi_inq_varid(ncid, "neighbors", &varids[21]); ERR; err = ncmpi_inq_var(ncid, varids[2], 0, &type, &ndims, dims, &natts); ERR; err = ncmpi_inq_dimlen(ncid, dims[0], &n); ERR; num_neighbors[b] = n; neighbors[b] = (int *)malloc(num_neighbors[b] * sizeof(int)); start[0] = start_block_ofst + b; count[0] = num_neighbors[b]; err = ncmpi_get_vara_int_all(ncid, varids[21], start, count, neighbors[b]); ERR; /* gids */ start[0] = start_block_ofst + b; count[0] = 1; err = ncmpi_inq_varid(ncid, "g_block_ids", &varids[22]); ERR; err = ncmpi_get_vara_int_all(ncid, varids[22], start, count, &gids[b]); ERR; (*vblocks)[b] = v; } /* cleanup */ err = ncmpi_close(ncid); ERR; free(block_ofsts); #endif }
int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, const char filename[], const int mode) { int ierr; int msg; int mpierr; size_t len; iosystem_desc_t *ios; file_desc_t *file; ierr = PIO_NOERR; msg = PIO_MSG_OPEN_FILE; ios = pio_get_iosystem_from_id(iosysid); if(ios==NULL){ printf("bad iosysid %d\n",iosysid); return PIO_EBADID; } file = (file_desc_t *) malloc(sizeof(*file)); if(file==NULL){ return PIO_ENOMEM; } file->iotype = *iotype; file->next = NULL; file->iosystem = ios; file->mode = mode; for(int i=0; i<PIO_MAX_VARS;i++){ file->varlist[i].record = -1; file->varlist[i].ndims = -1; #ifdef _PNETCDF file->varlist[i].request = NULL; file->varlist[i].nreqs=0; #endif file->varlist[i].fillbuf = NULL; file->varlist[i].iobuf = NULL; } file->buffer.validvars=0; file->buffer.vid=NULL; file->buffer.data=NULL; file->buffer.next=NULL; file->buffer.frame=NULL; file->buffer.fillvalue=NULL; if(ios->async_interface && ! ios->ioproc){ if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); len = strlen(filename); mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm); } if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: #ifdef _MPISERIAL ierr = nc_open(filename, file->mode, &(file->fh)); #else file->mode = file->mode | NC_MPIIO; ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh)); #endif break; case PIO_IOTYPE_NETCDF4C: file->mode = file->mode | NC_NETCDF4; // *** Note the INTENTIONAL FALLTHROUGH *** #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ ierr = nc_open(filename, file->mode, &(file->fh)); } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); // This should only be done with a file opened to append if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){ if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT); ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); } break; #endif default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); break; } // If we failed to open a file due to an incompatible type of NetCDF, try it // once with just plain old basic NetCDF #ifdef _NETCDF if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) { if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); // reset ierr on all tasks ierr = PIO_NOERR; // reset file markers for NETCDF on all tasks file->iotype = PIO_IOTYPE_NETCDF; // open netcdf file serially on main task if(ios->io_rank==0){ ierr = nc_open(filename, file->mode, &(file->fh)); } } #endif } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); if(ierr==PIO_NOERR){ mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm); pio_add_to_file_list(file); *ncidp = file->fh; } if(ios->io_rank==0){ printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); // if(file->fh==5) print_trace(stdout); } return ierr; }
int main(int argc, char ** argv) { int ncid, dimid, varid; MPI_Init(&argc, &argv); MPI_Datatype vtype, rtype, usertype; MPI_Aint lb, extent; int userbufsz, *userbuf, *cmpbuf, i, errs=0; int count = 25; double pi = 3.14159; MPI_Offset start, acount; ncmpi_create(MPI_COMM_WORLD, "vectors.nc", NC_CLOBBER, MPI_INFO_NULL, &ncid); ncmpi_def_dim(ncid, "50k", 1024*50, &dimid); ncmpi_def_var(ncid, "vector", NC_DOUBLE, 1, &dimid, &varid); ncmpi_enddef(ncid); MPI_Type_vector(VECCOUNT, BLOCKLEN, STRIDE, MPI_INT, &vtype); MPI_Type_create_resized(vtype, 0, STRIDE*VECCOUNT*sizeof(int), &rtype); MPI_Type_contiguous(count, rtype, &usertype); MPI_Type_commit(&usertype); MPI_Type_free(&vtype); MPI_Type_free(&rtype); MPI_Type_get_extent(usertype, &lb, &extent); userbufsz = extent; userbuf = malloc(userbufsz); cmpbuf = calloc(userbufsz, 1); for (i=0; i< userbufsz/sizeof(int); i++) { userbuf[i] = pi*i; } start = 10; acount = count*12; ncmpi_begin_indep_data(ncid); ncmpi_put_vara(ncid, varid, &start, &acount, userbuf, 1, usertype); ncmpi_close(ncid); NC_CHECK(ncmpi_open(MPI_COMM_WORLD, "vectors.nc", NC_NOWRITE, MPI_INFO_NULL, &ncid)); ncmpi_begin_indep_data(ncid); NC_CHECK(ncmpi_inq_varid(ncid, "vector", &varid)); NC_CHECK(ncmpi_get_vara(ncid, varid, &start, &acount, cmpbuf, 1, usertype)); ncmpi_close(ncid); for (i=0; errs < 10 && i < acount; i++) { /* vector of 4,3,5, so skip 4th and 5th items of every block */ if (i%STRIDE >= BLOCKLEN) continue; if (userbuf[i] != cmpbuf[i]) { errs++; fprintf(stderr, "%d: expected 0x%x got 0x%x\n", i, userbuf[i], cmpbuf[i]); } } free(userbuf); free(cmpbuf); MPI_Type_free(&usertype); MPI_Finalize(); return 0; }
/** * Read a single-precision parallel-nedcdf file. * * We assume here that localData is a scalar. * * Pnetcdf uses row-major format (same as FFTW). * * \param[in] filename : PnetCDF filename * \param[in] starts : offset to where to start reading data * \param[in] counts : number of elements read (3D sub-domain inside global) * \param[in] gsizes : global sizes * \param[out] localData : actual data buffer (size : nx*ny*nz*sizeof(float)) * * localData must have been allocated prior to calling this routine. */ void read_pnetcdf(const std::string &filename, MPI_Offset starts[3], MPI_Offset counts[3], int gsizes[3], float *localData) { int myRank; MPI_Comm_rank(MPI_COMM_WORLD, &myRank); // netcdf file id int ncFileId; int err; // file opening mode int ncOpenMode = NC_NOWRITE; int nbVar=1; int varIds[nbVar]; MPI_Info mpi_info_used; /* * Open NetCDF file */ err = ncmpi_open(MPI_COMM_WORLD, filename.c_str(), ncOpenMode, MPI_INFO_NULL, &ncFileId); if (err != NC_NOERR) { printf("Error: ncmpi_open() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err)); MPI_Abort(MPI_COMM_WORLD, -1); exit(1); } /* * Query NetCDF mode */ int NC_mode; err = ncmpi_inq_version(ncFileId, &NC_mode); if (myRank==0) { if (NC_mode == NC_64BIT_DATA) std::cout << "Pnetcdf Input mode : NC_64BIT_DATA (CDF-5)\n"; else if (NC_mode == NC_64BIT_OFFSET) std::cout << "Pnetcdf Input mode : NC_64BIT_OFFSET (CDF-2)\n"; else std::cout << "Pnetcdf Input mode : unknown\n"; } /* * Query information about variable named "data" */ { int ndims, nvars, ngatts, unlimited; err = ncmpi_inq(ncFileId, &ndims, &nvars, &ngatts, &unlimited); PNETCDF_HANDLE_ERROR; err = ncmpi_inq_varid(ncFileId, "data", &varIds[0]); PNETCDF_HANDLE_ERROR; } /* * Define expected data types (no conversion done here) */ MPI_Datatype mpiDataType = MPI_FLOAT; /* * Get all the MPI_IO hints used (just in case, we want to print it after * reading data... */ err = ncmpi_get_file_info(ncFileId, &mpi_info_used); PNETCDF_HANDLE_ERROR; /* * Read heavy data (take care of row-major / column major format !) */ int nItems = counts[IX]*counts[IY]*counts[IZ]; { err = ncmpi_get_vara_all(ncFileId, varIds[0], starts, counts, localData, nItems, mpiDataType); PNETCDF_HANDLE_ERROR; } // end reading heavy data /* * close the file */ err = ncmpi_close(ncFileId); PNETCDF_HANDLE_ERROR; } // read_pnetcdf
int main(int argc, char **argv) { int i, j; int status; int ncid1, ncid2; int ndims, nvars, ngatts, unlimdimid; char name[NC_MAX_NAME]; nc_type type, vartypes[NC_MAX_VARS]; MPI_Offset attlen; MPI_Offset dimlen, shape[NC_MAX_VAR_DIMS], varsize, start[NC_MAX_VAR_DIMS]; void *valuep; int dimids[NC_MAX_DIMS], varids[NC_MAX_VARS]; int vardims[NC_MAX_VARS][NC_MAX_VAR_DIMS/16]; /* divided by 16 due to my memory limitation */ int varndims[NC_MAX_VARS], varnatts[NC_MAX_VARS]; params opts; int rank; int nprocs; MPI_Comm comm = MPI_COMM_WORLD; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (rank == 0) fprintf(stderr, "Testing independent read ... "); parse_read_args(argc, argv, rank, &opts); /********** START OF NETCDF ACCESS **************/ /* Read a netCDF file and write it out to another file */ /** * Open the input dataset - ncid1: * File name: "../data/test_int.nc" * Dataset API: Collective * And create the output dataset - ncid2: * File name: "testread.nc" * Dataset API: Collective */ status = ncmpi_open(comm, opts.infname, 0, MPI_INFO_NULL, &ncid1); if (status != NC_NOERR) handle_error(status); status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid2); if (status != NC_NOERR) handle_error(status); /** * Inquire the dataset definitions of input dataset AND * Add dataset definitions for output dataset. */ status = ncmpi_inq(ncid1, &ndims, &nvars, &ngatts, &unlimdimid); if (status != NC_NOERR) handle_error(status); /* Inquire global attributes, assume CHAR attributes. */ for (i = 0; i < ngatts; i++) { status = ncmpi_inq_attname(ncid1, NC_GLOBAL, i, name); if (status != NC_NOERR) handle_error(status); status = ncmpi_inq_att (ncid1, NC_GLOBAL, name, &type, &attlen); if (status != NC_NOERR) handle_error(status); switch (type) { case NC_CHAR: valuep = (void *)malloc(attlen * sizeof(char)); status = ncmpi_get_att_text(ncid1, NC_GLOBAL, name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_text (ncid2, NC_GLOBAL, name, attlen, (char *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_SHORT: valuep = (void *)malloc(attlen * sizeof(short)); status = ncmpi_get_att_short(ncid1, NC_GLOBAL, name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_short (ncid2, NC_GLOBAL, name, type, attlen, (short *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_INT: valuep = (void *)malloc(attlen * sizeof(int)); status = ncmpi_get_att_int(ncid1, NC_GLOBAL, name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_int (ncid2, NC_GLOBAL, name, type, attlen, (int *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_FLOAT: valuep = (void *)malloc(attlen * sizeof(float)); status = ncmpi_get_att_float(ncid1, NC_GLOBAL, name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_float (ncid2, NC_GLOBAL, name, type, attlen, (float *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_DOUBLE: valuep = (void *)malloc(attlen * sizeof(double)); status = ncmpi_get_att_double(ncid1, NC_GLOBAL, name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_double (ncid2, NC_GLOBAL, name, type, attlen, (double *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; default: ; /* handle unexpected types */ } } /* Inquire dimension */ for (i = 0; i < ndims; i++) { status = ncmpi_inq_dim(ncid1, i, name, &dimlen); if (status != NC_NOERR) handle_error(status); if (i == unlimdimid) dimlen = NC_UNLIMITED; status = ncmpi_def_dim(ncid2, name, dimlen, dimids+i); if (status != NC_NOERR) handle_error(status); } /* Inquire variables */ for (i = 0; i < nvars; i++) { status = ncmpi_inq_var (ncid1, i, name, vartypes+i, varndims+i, vardims[i], varnatts+i); if (status != NC_NOERR) handle_error(status); status = ncmpi_def_var(ncid2, name, vartypes[i], varndims[i], vardims[i], varids+i); if (status != NC_NOERR) handle_error(status); /* var attributes, assume CHAR attributes */ for (j = 0; j < varnatts[i]; j++) { status = ncmpi_inq_attname(ncid1, varids[i], j, name); if (status != NC_NOERR) handle_error(status); status = ncmpi_inq_att (ncid1, varids[i], name, &type, &attlen); if (status != NC_NOERR) handle_error(status); switch (type) { case NC_CHAR: valuep = (void *)malloc(attlen * sizeof(char)); status = ncmpi_get_att_text(ncid1, varids[i], name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_text (ncid2, varids[i], name, attlen, (char *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_SHORT: valuep = (void *)malloc(attlen * sizeof(short)); status = ncmpi_get_att_short(ncid1, varids[i], name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_short (ncid2, varids[i], name, type, attlen, (short *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_INT: valuep = (void *)malloc(attlen * sizeof(int)); status = ncmpi_get_att_int(ncid1, varids[i], name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_int (ncid2, varids[i], name, type, attlen, (int *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_FLOAT: valuep = (void *)malloc(attlen * sizeof(float)); status = ncmpi_get_att_float(ncid1, varids[i], name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_float (ncid2, varids[i], name, type, attlen, (float *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_DOUBLE: valuep = (void *)malloc(attlen * sizeof(double)); status = ncmpi_get_att_double(ncid1, varids[i], name, valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_att_double (ncid2, varids[i], name, type, attlen, (double *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; default: ; /* handle unexpected types */ } } } /** * End Define Mode (switch to data mode) for output dataset * Dataset API: Collective */ status = ncmpi_enddef(ncid2); if (status != NC_NOERR) handle_error(status); /** * Read data of variables from input dataset (assume INT variables) * Write the data out to the corresponding variables in the output dataset * * Data Partition (Assume 4 processors): * square: 2-D, (Block, *), 25*100 from 100*100 * cube: 3-D, (Block, *, *), 25*100*100 from 100*100*100 * xytime: 3-D, (Block, *, *), 25*100*100 from 100*100*100 * time: 1-D, Block-wise, 25 from 100 * * Data Mode API: non-collective */ status = ncmpi_begin_indep_data(ncid1); if (status != NC_NOERR) handle_error(status); status =ncmpi_begin_indep_data(ncid2); if (status != NC_NOERR) handle_error(status); for (i = 0; i < NC_MAX_VAR_DIMS; i++) start[i] = 0; for (i = 0; i < nvars; i++) { varsize = 1; for (j = 0; j < varndims[i]; j++) { status = ncmpi_inq_dim(ncid1, vardims[i][j], name, shape + j); if (status != NC_NOERR) handle_error(status); if (j == 0) { shape[j] /= nprocs; start[j] = shape[j] * rank; } varsize *= shape[j]; } switch (vartypes[i]) { case NC_CHAR: break; case NC_SHORT: valuep = (void *)malloc(varsize * sizeof(short)); status = ncmpi_get_vara_short(ncid1, i, start, shape, (short *)valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_vara_short(ncid2, varids[i], start, shape, (short *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_INT: valuep = (void *)malloc(varsize * sizeof(int)); status = ncmpi_get_vara_int(ncid1, i, start, shape, (int *)valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_vara_int(ncid2, varids[i], start, shape, (int *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_FLOAT: valuep = (void *)malloc(varsize * sizeof(float)); status = ncmpi_get_vara_float(ncid1, i, start, shape, (float *)valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_vara_float(ncid2, varids[i], start, shape, (float *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; case NC_DOUBLE: valuep = (void *)malloc(varsize * sizeof(double)); status = ncmpi_get_vara_double(ncid1, i, start, shape, (double *)valuep); if (status != NC_NOERR) handle_error(status); status = ncmpi_put_vara_double(ncid2, varids[i], start, shape, (double *)valuep); if (status != NC_NOERR) handle_error(status); free(valuep); break; default: ; /* handle unexpected types */ } } status = ncmpi_end_indep_data(ncid1); if (status != NC_NOERR) handle_error(status); status = ncmpi_end_indep_data(ncid2); if (status != NC_NOERR) handle_error(status); status = ncmpi_sync(ncid1); if (status != NC_NOERR) handle_error(status); status = ncmpi_sync(ncid2); if (status != NC_NOERR) handle_error(status); /** * Close the datasets * Dataset API: collective */ status = ncmpi_close(ncid1); if (status != NC_NOERR) handle_error(status); status = ncmpi_close(ncid2); if (status != NC_NOERR) handle_error(status); /******************* END OF NETCDF ACCESS ****************/ if (rank == 0) fprintf(stderr, "OK\nInput file %s copied to: %s!\n", opts.infname, opts.outfname); MPI_Finalize(); return 0; }
int main(int argc, char **argv) { int i, j=0, rank, nprocs, err; int ncfile, ndims, nvars, ngatts, unlimited, var_ndims, var_natts;; int *dimids=NULL; char filename[256], varname[NC_MAX_NAME+1]; MPI_Offset *dim_sizes=NULL, var_size; nc_type type; int *data=NULL; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (argc > 2) { if (rank == 0) printf("Usage: %s filename\n", argv[0]); MPI_Finalize(); exit(-1); } if (argc > 1) snprintf(filename, 256, "%s", argv[1]); else strcpy(filename, "testfile.nc"); if (rank == 0) { err = ncmpi_open(MPI_COMM_SELF, filename, NC_NOWRITE, MPI_INFO_NULL, &ncfile); if (err != NC_NOERR) handle_error(err, __LINE__); /* reader knows nothing about dataset, but we can interrogate with * query routines: ncmpi_inq tells us how many of each kind of * "thing" (dimension, variable, attribute) we will find in the * file */ err = ncmpi_inq(ncfile, &ndims, &nvars, &ngatts, &unlimited); if (err != NC_NOERR) handle_error(err, __LINE__); /* we do not really need the name of the dimension or the variable * for reading in this example. we could, in a different example, * take the name of a variable on the command line and read just * that one */ dim_sizes = (MPI_Offset*) calloc(ndims, sizeof(MPI_Offset)); /* netcdf dimension identifiers are allocated sequentially starting * at zero; same for variable identifiers */ for(i=0; i<ndims; i++) { err = ncmpi_inq_dimlen(ncfile, i, &(dim_sizes[i]) ); if (err != NC_NOERR) handle_error(err, __LINE__); } } /* need to inform other MPI processors how many variables we will send */ err = MPI_Bcast(&nvars, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_ERR(err) for(i=0; i<nvars; i++) { /* rank 0 will find out the size of each variable, read it, and * broadcast it to the rest of the processors */ if (rank == 0) { /* obtain the number of dimensions of variable i, so we can * allocate the dimids array */ err = ncmpi_inq_varndims(ncfile, i, &var_ndims); if (err != NC_NOERR) handle_error(err, __LINE__); dimids = (int*) malloc(var_ndims * sizeof(int)); err = ncmpi_inq_var(ncfile, i, varname, &type, &var_ndims, dimids, &var_natts); if (err != NC_NOERR) handle_error(err, __LINE__); for (j=0, var_size=1; j<var_ndims; j++) { var_size *= dim_sizes[dimids[j]]; } free(dimids); } /* oddity: there's no predefined MPI_Offset type */ err = MPI_Bcast(&var_size, 1, MPI_OFFSET, 0, MPI_COMM_WORLD); MPI_ERR(err) data = (int*) calloc(var_size, sizeof(int)); if (rank == 0) { switch(type) { case NC_INT: /* now we have the variable identifiers and we know how big * they are */ /* this approach is not scalable: i/o happens from a single * processor. This approach can be ok if the amount of * data is quite small, but almost always the underlying * MPI-IO library can do a better job */ err = ncmpi_get_var_int_all(ncfile, j, data); if (err != NC_NOERR) handle_error(err, __LINE__); break; default: /* we can do this for all the known netcdf types but this * example is already getting too long */ fprintf(stderr, "unsupported NetCDF type \n"); } } /*and finally all processors have the data */ err = MPI_Bcast(data, var_size, MPI_INT, 0, MPI_COMM_WORLD); MPI_ERR(err) /* Here, every process can do computation on the local buffer, data, or copy the contents to somewhere else */ free(data); } if (rank == 0) { err = ncmpi_close(ncfile); if (err != NC_NOERR) handle_error(err, __LINE__); free(dim_sizes); } MPI_Finalize(); return 0; }