/* Test a small file with two record vars, which grow, and has
* attributes added. */
static int
test_two_growing_with_att(const char *testfile, int cmode)
{
int err, ncid, dimid, varid[NUM_VARS];
char data[MAX_RECS], data_in;
char att_name[NC_MAX_NAME + 1];
MPI_Offset start[ONE_DIM], count[ONE_DIM], index[ONE_DIM], len_in;
int v, r;
/* Create a file with one ulimited dimensions, and one var. */
err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid); ERR
err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid[0]); ERR
err=ncmpi_def_var(ncid, VAR_NAME2, NC_CHAR, 1, &dimid, &varid[1]); ERR
err=ncmpi_close(ncid); ERR
/* Create some phoney data. */
for (data[0] = 'a', r = 1; r < MAX_RECS; r++)
data[r] = data[r - 1] + 1;
/* Normally one would not close and reopen the file for each
* record, nor add an attribute each time I add a record, but I am
* giving the library a little work-out here... */
for (r = 0; r < MAX_RECS; r++)
{
/* Write one record of var data, a single character. */
err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_WRITE, MPI_INFO_NULL, &ncid); ERR
count[0] = 1;
start[0] = r;
sprintf(att_name, "a_%d", data[r]);
for (v = 0; v < NUM_VARS; v++)
{
err=ncmpi_put_vara_text_all(ncid, varid[v], start, count, &data[r]); ERR
err=ncmpi_redef(ncid); ERR
err=ncmpi_put_att_text(ncid, varid[v], att_name, 1, &data[r]); ERR
err=ncmpi_enddef(ncid); ERR
}
err=ncmpi_close(ncid); ERR
/* Reopen the file and check it. */
err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_inq_dimlen(ncid, 0, &len_in); ERR
if (len_in != r + 1) {printf("Error at line %d\n",__LINE__);return 1;}
index[0] = r;
err=ncmpi_begin_indep_data(ncid); ERR
for (v = 0; v < NUM_VARS; v++)
{
err=ncmpi_get_var1_text(ncid, varid[v], index, &data_in); ERR
if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;}
}
err=ncmpi_close(ncid); ERR
} /* Next record. */
return 0;
}
/* Test a small file with one var and one att. */
static int
test_one_with_att(const char *testfile, int cmode)
{
int err, ncid, dimid, varid;
char data = 'h', data_in;
int ndims, nvars, natts, unlimdimid;
MPI_Offset start[NDIMS], count[NDIMS];
/* Create a file with one ulimited dimensions, and one var. */
err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid); ERR
err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid); ERR
err=ncmpi_put_att_text(ncid, NC_GLOBAL, ATT_NAME, 1, &data); ERR
err=ncmpi_enddef(ncid); ERR
/* Write one record of var data, a single character. */
count[0] = 1;
start[0] = 0;
err=ncmpi_put_vara_text_all(ncid, varid, start, count, &data); ERR
/* We're done! */
err=ncmpi_close(ncid); ERR
/* Reopen the file and check it. */
err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR
if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_get_var_text_all(ncid, varid, &data_in); ERR
if (data_in != data) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_get_att_text(ncid, NC_GLOBAL, ATT_NAME, &data_in); ERR
if (data_in != data) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_close(ncid); ERR
return 0;
}
/* Test a small file with one record var, which grows. */
static int
test_one_growing(const char *testfile, int cmode)
{
int err, ncid, dimid, varid;
char data[MAX_RECS], data_in;
MPI_Offset start[ONE_DIM], count[ONE_DIM], index[ONE_DIM], len_in;
int r, f;
/* Create some phoney data. */
for (data[0] = 'a', r = 1; r < MAX_RECS; r++)
data[r] = data[r - 1] + 1;
/* Run this with and without fill mode. */
for (f = 0; f < 2; f++)
{
/* Create a file with one ulimited dimensions, and one var. */
err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid); ERR
err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid); ERR
err=ncmpi_close(ncid); ERR
/* Normally one would not close and reopen the file for each
* record, but I am giving the library a little work-out here... */
for (r = 0; r < MAX_RECS; r++)
{
/* Write one record of var data, a single character. */
err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_WRITE, MPI_INFO_NULL, &ncid); ERR
/* if (f) { err=ncmpi_set_fill(ncid, NC_NOFILL, NULL); ERR} */
count[0] = 1;
start[0] = r;
err=ncmpi_put_vara_text_all(ncid, varid, start, count, &data[r]); ERR
err=ncmpi_close(ncid); ERR
/* Reopen the file and check it. */
err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_inq_dimlen(ncid, 0, &len_in); ERR
if (len_in != r + 1) {printf("Error at line %d\n",__LINE__);return 1;}
index[0] = r;
err=ncmpi_begin_indep_data(ncid); ERR
err=ncmpi_get_var1_text(ncid, 0, index, &data_in); ERR
if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_close(ncid); ERR
} /* Next record. */
}
return 0;
}
int main(int argc, char** argv)
{
extern int optind;
char *filename="testfile.nc";
int i, rank, verbose=1, err;
int ncid, cmode, omode;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* get command-line arguments */
while ((i = getopt(argc, argv, "hq")) != EOF)
switch(i) {
case 'q': verbose = 0;
break;
case 'h':
default: if (rank==0) usage(argv[0]);
MPI_Finalize();
return 0;
}
argc -= optind;
argv += optind;
if (argc == 1) filename = argv[0]; /* optional argument */
if (verbose && rank == 0) printf("%s: example of file create and open\n",__FILE__);
/* create a new file using clobber mode ----------------------------------*/
cmode = NC_CLOBBER;
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid);
ERR
/* close file */
err = ncmpi_close(ncid);
ERR
/* open the newly created file for read only -----------------------------*/
omode = NC_NOWRITE;
err = ncmpi_open(MPI_COMM_WORLD, filename, omode, MPI_INFO_NULL, &ncid);
ERR
/* close file */
err = ncmpi_close(ncid);
ERR
/* check if there is any PnetCDF internal malloc residue */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
}
MPI_Finalize();
return 0;
}
int Traj_AmberNetcdf::parallelOpenTrajout(Parallel::Comm const& commIn) {
if (Ncid() != -1) return 0;
int err = ncmpi_open(commIn.MPIcomm(), filename_.full(), NC_WRITE, MPI_INFO_NULL, &ncid_);
if (checkPNCerr(err)) {
mprinterr("Error: Opening NetCDF file '%s' for writing in parallel.\n", filename_.full());
return 1;
}
err = ncmpi_begin_indep_data( ncid_ ); // Independent data mode
return 0;
}
static int
NC5_open(const char *path, int cmode,
int basepe, size_t *chunksizehintp,
int use_parallel, void* mpidata,
struct NC_Dispatch* table, NC* nc)
{
int res;
NC5_INFO* nc5;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
/* Check the cmode for only valid flags*/
if(cmode & ~LEGAL_OPEN_FLAGS)
return NC_EINVAL;
/* Cannot have both MPIO flags */
if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX))
return NC_EINVAL;
/* Appears that this comment is wrong; allow 64 bit offset*/
/* Cannot have 64 bit offset flag */
/* if(cmode & (NC_64BIT_OFFSET)) return NC_EINVAL; */
if(mpidata != NULL) {
comm = ((NC_MPI_INFO *)mpidata)->comm;
info = ((NC_MPI_INFO *)mpidata)->info;
} else {
comm = MPI_COMM_WORLD;
info = MPI_INFO_NULL;
}
/* Fix up the cmode by keeping only essential flags;
these are the flags that are the same in netcf.h and pnetcdf.h
*/
cmode &= (NC_WRITE | NC_NOCLOBBER | NC_LOCK | NC_SHARE | NC_64BIT_OFFSET);
cmode |= (NC_NETCDF4); /* see comment in NC5_create */
/* Create our specific NC5_INFO instance */
nc5 = (NC5_INFO*)calloc(1,sizeof(NC5_INFO));
if(nc5 == NULL) return NC_ENOMEM;
/* Link nc5 and nc */
NC5_DATA_SET(nc,nc5);
res = ncmpi_open(comm, path, cmode, info, &(nc->int_ncid));
/* Default to independent access, like netCDF-4/HDF5 files. */
if(!res) {
res = ncmpi_begin_indep_data(nc->int_ncid);
nc5->pnetcdf_access_mode = NC_INDEPENDENT;
}
return res;
}
//----------------------------------------------------------------
// Open the first NETCDF file and query all the types for all the variables
static void determine_var_types()
{
assert(netcdf_file_names != 0);
assert(netcdf_var_names != 0);
struct Type type;
int plist_id,i = 0,varidp,ndimsp;
int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[0], NC_NOWRITE, MPI_INFO_NULL, &plist_id);
if (file_id != NC_NOERR)
terminate_with_error_msg("ERROR: Cannot open file %s\n", netcdf_file_names[0]);
var_types = (struct Type *)calloc(var_count, sizeof(*var_types));
nc_type xtypep;
for (i = 0; i < var_count; ++i)
{
int dataset_id = ncmpi_inq_varid (plist_id, netcdf_var_names[i], &varidp);
dataset_id = ncmpi_inq_vartype(plist_id,varidp, &xtypep);
if (dataset_id != NC_NOERR)
terminate_with_error_msg("ERROR: Cannot read the datatype of the variable %s\n", netcdf_var_names[i]);
int num_dims = ncmpi_inq_varndims(plist_id,varidp,&ndimsp);
if (ndimsp > 3 )
{
//TODO:probably we can make it more clever to handle more dimensions as they are not related to the variable itself on netcdf
type.atomic_type = INVALID; // we don't support arrays of more than 3 dimension
return;
}
else if (xtypep == NC_FLOAT || xtypep == NC_INT)
{
type.num_values = 1;
}
else // we don't support HD5_COMPOUND datatype for example
{
type.atomic_type = INVALID;
}
type.atomic_type = from_netcdf_atomic_type(xtypep);
var_types[i] = type;
if (var_types[i].atomic_type == INVALID)
terminate_with_error_msg("ERROR: The datatype of the %s variable is not supported\n", netcdf_var_names[i]);
}
ncmpi_close(plist_id);
}
// Traj_NcEnsemble::openTrajin()
int Traj_NcEnsemble::openTrajin() {
// If already open, return
if (Ncid()!=-1) return 0;
# if HAS_PNETCDF
int err = ncmpi_open(MPI_COMM_WORLD, filename_.full(), NC_NOWRITE, MPI_INFO_NULL, &ncid_);
//err += ncmpi_begin_indep_data( ncid_ ); // Disable independent data mode
# else
int err = NC_openRead( filename_.Full() );
# endif
if ( err != 0 ) {
mprinterr("Error: Opening Netcdf file %s for reading.\n", filename_.base());
return 1;
}
return 0;
}
/* Test a small file with an unlimited dimension. NOTE: Normally I
* write a NULL terminator for my attributes and text strings, but
* this reproduces a bug that a fortran user sent us. So string data
* are written to the file without null terminators. - Ed */
static int
test_small_unlim(const char *testfile, int cmode)
{
int i, err, ncid, dimids[NDIMS], varid;
char data[NUM_VALS][STR_LEN + 1], data_in[NUM_VALS][STR_LEN];
int ndims, nvars, natts, unlimdimid;
MPI_Offset start[NDIMS], count[NDIMS];
/* Create null-terminated text strings of correct length. */
/*for (i = 0; i < NUM_VALS; i++)
strcpy(data[i], source);*/
strcpy(data[0], "2005-04-11_12:00:00");
strcpy(data[1], "2005-04-11_13:00:00");
/* Create a file with two dimensions, one unlimited, and one
* var, and a global att. */
err=ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, dimids); ERR
err=ncmpi_def_dim(ncid, DIM2_NAME, STR_LEN, &dimids[1]); ERR
err=ncmpi_def_var(ncid, VAR_NAME, NC_CHAR, 2, dimids, &varid); ERR
err=ncmpi_put_att_text(ncid, NC_GLOBAL, ATT_NAME2, strlen(TITLE), TITLE); ERR
err=ncmpi_enddef(ncid); ERR
/* Write some records of var data. */
count[0] = 1;
count[1] = STR_LEN;
start[1] = 0;
for (start[0] = 0; start[0] < NUM_VALS; start[0]++) {
err=ncmpi_put_vara_text_all(ncid, varid, start, count, data[start[0]]); ERR
}
/* We're done! */
err=ncmpi_close(ncid); ERR
/* Reopen the file and check it. */
err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR
if (ndims != 2 && nvars != 1 && natts != 0 && unlimdimid != 0) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_get_var_text_all(ncid, varid, (char *)data_in); ERR
for (i = 0; i < NUM_VALS; i++)
/* if (strncmp(data[i], data_in[i], STR_LEN)) {printf("Error at line %d\n",__LINE__);return 1;} */
if (strncmp(data[i], data_in[i], STR_LEN)) {
printf("i=%d data=%s data_in=%s\n",i,data[i],data_in[i]);
}
err=ncmpi_close(ncid); ERR
return 0;
}
void BIL_Pio_read_nc_blocks(MPI_Comm all_readers_comm, MPI_Comm io_comm,
int num_blocks, BIL_Block* blocks) {
int i;
for (i = 0; i < num_blocks; i++) {
int fp;
BIL_Timing_fopen_start(all_readers_comm);
assert(ncmpi_open(io_comm, blocks[i].file_name, NC_NOWRITE,
BIL->io_hints, &fp) == NC_NOERR);
BIL_Timing_fopen_stop(all_readers_comm);
ncmpi_begin_indep_data(fp);
// Find the id, type, and size of the variable.
int var_id;
assert(ncmpi_inq_varid(fp, blocks[i].var_name, &var_id) == NC_NOERR);
nc_type var_type;
assert(ncmpi_inq_vartype(fp, var_id, &var_type) == NC_NOERR);
// Create extra variables specifically for the netCDF API.
MPI_Offset nc_dim_starts[BIL_MAX_NUM_DIMS];
MPI_Offset nc_dim_sizes[BIL_MAX_NUM_DIMS];
int j;
for (j = 0; j < blocks[i].num_dims; j++) {
nc_dim_starts[j] = blocks[i].starts[j];
nc_dim_sizes[j] = blocks[i].sizes[j];
}
MPI_Datatype nc_var_type;
BIL_Pio_nc_to_mpi_type(var_type, &nc_var_type, &(blocks[i].var_size));
// Allocate room for data and read it independently.
blocks[i].data = BIL_Misc_malloc(blocks[i].total_size * blocks[i].var_size);
BIL_Timing_io_start(all_readers_comm);
assert(ncmpi_get_vara(fp, var_id, nc_dim_starts, nc_dim_sizes,
blocks[i].data, blocks[i].total_size,
nc_var_type) == NC_NOERR);
BIL_Timing_io_stop(all_readers_comm,
blocks[i].total_size * blocks[i].var_size);
// Clean up.
ncmpi_end_indep_data(fp);
ncmpi_close(fp);
}
}
void *
IOR_Open_NCMPI(char * testFileName,
IOR_param_t * param)
{
int * fd;
int fd_mode;
MPI_Info mpiHints = MPI_INFO_NULL;
/* Wei-keng Liao: read and set MPI file hints from hintsFile */
SetHints(&mpiHints, param->hintsFileName);
if (rank == 0 && param->showHints) {
fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
fd = (int *)malloc(sizeof(int));
if (fd == NULL) ERR("Unable to malloc file descriptor");
fd_mode = GetFileMode(param);
NCMPI_CHECK(ncmpi_open(testComm, testFileName, fd_mode,
mpiHints, fd), "cannot open file");
/* Wei-keng Liao: print the MPI file hints currently used */
/* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release
if (rank == 0 && param->showHints) {
MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints),
"cannot get file info");
fprintf(stdout, "\nhints returned from opened file {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
*/
/* Wei-keng Liao: free up the mpiHints object */
/* WEL - this needs future fix from next release of PnetCDF
if (mpiHints != MPI_INFO_NULL)
MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
*/
return(fd);
} /* IOR_Open_NCMPI() */
static int
test_small_atts(const char *testfile, int cmode)
{
int ncid, err;
char att[MAX_LEN + 1], att_in[MAX_LEN + 1], source[MAX_LEN + 1] = "0123456";
int ndims, nvars, natts, unlimdimid;
MPI_Offset len_in;
int t, f;
/* Run this with and without fill mode. */
for (f = 0; f < 2; f++)
{
/* Create small files with an attribute that grows by one each
* time. */
for (t = 1; t < MAX_LEN; t++)
{
/* Create null-terminated text string of correct length. */
strncpy(att, source, t);
att[t] = '\0';
/* Create a file with one attribute. */
err = ncmpi_create(MPI_COMM_WORLD, testfile,cmode, MPI_INFO_NULL, &ncid); ERR
err = ncmpi_put_att_text(ncid, NC_GLOBAL, ATT_NAME, t + 1, att); ERR
if (f) { err=ncmpi_set_fill(ncid, NC_NOFILL, NULL); ERR}
err=ncmpi_close(ncid); ERR;
/* Reopen the file and check it. */
err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR
err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR
if (ndims != 0 && nvars != 0 && natts != 1 && unlimdimid != -1) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &len_in); ERR
if (len_in != t + 1) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_get_att_text(ncid, NC_GLOBAL, ATT_NAME, att_in); ERR
if (strncmp(att_in, att, t)) {printf("Error at line %d\n",__LINE__);return 1;}
err=ncmpi_close(ncid); ERR
}
}
return 0;
}
// Traj_NcEnsemble::setupTrajout()
int Traj_NcEnsemble::setupTrajout(FileName const& fname, Topology* trajParm,
CoordinateInfo const& cInfoIn,
int NframesToWrite, bool append)
{
int err = 0;
# ifdef MPI
if (NoPnetcdf()) return 1;
# endif
readAccess_ = false;
if (!append) {
CoordinateInfo cInfo = cInfoIn;
// TODO: File output modifications
SetCoordInfo( cInfo );
# ifdef MPI
ensembleStart_ = Parallel::World().Rank();
ensembleEnd_ = Parallel::World().Rank() + 1;
# else
ensembleStart_ = 0;
ensembleEnd_ = cInfo.EnsembleSize();;
# endif
filename_ = fname;
// Set up title
if (Title().empty())
SetTitle("Cpptraj Generated trajectory");
# ifdef MPI
if (Parallel::World().Master()) { // Only master creates file.
# endif
// Create NetCDF file.
err = NC_create(filename_.Full(), NC_AMBERENSEMBLE, trajParm->Natom(), CoordInfo(),
Title(), debug_);
// Close Netcdf file. It will be reopened write. FIXME should NC_create leave it closed?
NC_close();
# ifdef MPI
}
Parallel::World().MasterBcast(&err, 1, MPI_INT);
# endif
if (err != 0) return 1;
# ifdef MPI
// Synchronize netcdf info on non-master threads
Sync(Parallel::World());
// DEBUG: Print info for all ranks
DebugVIDs();
# endif
// Allocate memory
if (Coord_!=0) delete[] Coord_;
Coord_ = new float[ Ncatom3() ];
} else { // NOTE: File existence is checked for in Trajout
// Call setupTrajin to set input parameters. This will also allocate
// memory for coords.
if (setupTrajin(fname, trajParm) == TRAJIN_ERR) return 1;
if (debug_ > 0)
mprintf("\tNetCDF: Appending %s starting at frame %i\n", filename_.base(), Ncframe());
}
// Open file
# ifdef HAS_PNETCDF
err = ncmpi_open(MPI_COMM_WORLD, filename_.full(), NC_WRITE, MPI_INFO_NULL, &ncid_);
// TODO: Graceful error handling
# else
err = NC_openWrite( filename_.Full() );
# endif
if ( err != 0 ) {
mprinterr("Error: Opening Netcdf file %s for Write.\n", filename_.base());
return 1;
}
return 0;
}
int main(int argc, char** argv)
{
char filename[256];
int i, j, rank, nprocs, err, nerrs=0, expected;
int ncid, cmode, varid[2], dimid[2], req[4], st[4], *buf;
int *buf0, *buf1, *buf2;
size_t len;
MPI_Offset start[2], count[2];
MPI_Info info;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* this program is intended to run on one process */
if (rank) goto fn_exit;
/* get command-line arguments */
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for writing interleaved fileviews ", basename(argv[0]));
printf("%-66s ------ ", cmd_str);
free(cmd_str);
}
MPI_Info_create(&info);
MPI_Info_set(info, "romio_cb_write", "disable");
MPI_Info_set(info, "ind_wr_buffer_size", "8");
/* these 2 hints are required to cause a core dump if r1758 fix is not
* presented */
/* create a new file for writing ----------------------------------------*/
cmode = NC_CLOBBER | NC_64BIT_DATA;
err = ncmpi_create(MPI_COMM_SELF, filename, cmode, info, &ncid); CHECK_ERR
MPI_Info_free(&info);
/* define dimensions Y and X */
err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR
/* define 2D variables of integer type */
err = ncmpi_def_var(ncid, "var0", NC_INT, 2, dimid, &varid[0]); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 2, dimid, &varid[1]); CHECK_ERR
/* enable fill mode */
err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR
/* do not forget to exit define mode */
err = ncmpi_enddef(ncid); CHECK_ERR
/* now we are in data mode */
buf = (int*) malloc(NY*NX * sizeof(int));
/* fill the entire variable var0 with -1s */
for (i=0; i<NY*NX; i++) buf[i] = -1;
err = ncmpi_put_var_int_all(ncid, varid[0], buf); CHECK_ERR
/* write 8 x 2 elements so this only interleaves the next two
* iput requests */
start[0] = 0; start[1] = 3;
count[0] = 8; count[1] = 2;
len = (size_t)(count[0] * count[1]);
buf0 = (int*) malloc(len * sizeof(int));
for (i=0; i<len; i++) buf0[i] = 50+i;
err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf0, &req[0]);
CHECK_ERR
/* write 1 x 3 elements */
start[0] = 1; start[1] = 8;
count[0] = 1; count[1] = 5;
len = (size_t)(count[0] * count[1]);
buf1 = (int*) malloc(len * sizeof(int));
for (i=0; i<len; i++) buf1[i] = 60+i;
err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf1, &req[1]);
CHECK_ERR
/* write 1 x 3 elements */
start[0] = 3; start[1] = 7;
count[0] = 1; count[1] = 5;
len = (size_t)(count[0] * count[1]);
buf2 = (int*) malloc(len * sizeof(int));
for (i=0; i<len; i++) buf2[i] = 70+i;
err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf2, &req[2]);
CHECK_ERR
err = ncmpi_wait_all(ncid, 3, req, st); CHECK_ERR
free(buf0); free(buf1); free(buf2);
/* fill the entire variable var1 with -1s */
for (i=0; i<NY*NX; i++) buf[i] = -1;
err = ncmpi_put_var_int_all(ncid, varid[1], buf); CHECK_ERR
/* write 8 x 2 elements so this only interleaves the next two iput
* requests */
start[0] = 0; start[1] = 3;
count[0] = 8; count[1] = 2;
len = (size_t)(count[0] * count[1]);
buf0 = (int*) malloc(len * sizeof(int));
for (i=0; i<count[0]*count[1]; i++) buf0[i] = 50+i;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf0, &req[0]);
CHECK_ERR
/* rearrange buffer contents, as buf is 2D */
for (i=0; i<5; i++) buf[i] = 10 + i;
for (i=5; i<10; i++) buf[i] = 10 + i + 5;
for (i=10; i<15; i++) buf[i] = 10 + i + 10;
start[0] = 6; start[1] = 7;
count[0] = 3; count[1] = 5;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf, &req[1]);
CHECK_ERR
for (i=15; i<20; i++) buf[i] = 10 + i - 10;
for (i=20; i<25; i++) buf[i] = 10 + i - 5;
start[0] = 6; start[1] = 12;
count[0] = 2; count[1] = 5;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf+15, &req[2]);
CHECK_ERR
for (i=25; i<30; i++) buf[i] = 10 + i;
start[0] = 8; start[1] = 12;
count[0] = 1; count[1] = 5;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf+25, &req[3]);
CHECK_ERR
err = ncmpi_wait_all(ncid, 4, req, st); CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<16; i++) CHECK_CONTENTS(buf0, 50 + i)
for (i=0; i<5; i++) CHECK_CONTENTS(buf, 10 + i)
for (i=5; i<10; i++) CHECK_CONTENTS(buf, 10 + i + 5)
for (i=10; i<15; i++) CHECK_CONTENTS(buf, 10 + i + 10)
for (i=15; i<20; i++) CHECK_CONTENTS(buf, 10 + i - 10)
for (i=20; i<25; i++) CHECK_CONTENTS(buf, 10 + i - 5)
for (i=25; i<30; i++) CHECK_CONTENTS(buf, 10 + i)
err = ncmpi_close(ncid); CHECK_ERR
free(buf0);
/* open the same file and read back for validate */
err = ncmpi_open(MPI_COMM_SELF, filename, NC_NOWRITE, MPI_INFO_NULL,
&ncid); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var0", &varid[0]); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var1", &varid[1]); CHECK_ERR
/* read the entire array */
for (i=0; i<NY*NX; i++) buf[i] = -1;
err = ncmpi_get_var_int_all(ncid, varid[0], buf); CHECK_ERR
/* check if the contents of buf are expected */
expected = 50;
for (j=0; j<8; j++) {
for (i=3; i<5; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
}
expected = 60;
j = 1;
for (i=8; i<13; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
expected = 70;
j = 3;
for (i=7; i<12; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
/* initialize the contents of the array to a different value */
for (i=0; i<NY*NX; i++) buf[i] = -1;
/* read the entire array */
err = ncmpi_get_var_int_all(ncid, varid[1], buf); CHECK_ERR
/* check if the contents of buf are expected */
expected = 10;
for (j=6; j<9; j++) {
for (i=7; i<17; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d]=%d, should be %d\n",
rank, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
}
expected = 50;
for (j=0; j<8; j++) {
for (i=3; i<5; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
}
err = ncmpi_close(ncid); CHECK_ERR
free(buf);
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR && malloc_size > 0) {
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", malloc_size);
ncmpi_inq_malloc_list();
}
fn_exit:
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
static
int tst_norm(char *filename, int cmode)
{
int ncid, dimid, varid;
int dimids[NDIMS];
/* unnormalized UTF-8 encoding for Unicode 8-character "Hello" in Greek: */
unsigned char uname_utf8[] = {
0x41, /* LATIN CAPITAL LETTER A */
0xCC, 0x80, /* COMBINING GRAVE ACCENT */
0x41, /* LATIN CAPITAL LETTER A */
0xCC, 0x81, /* COMBINING ACUTE ACCENT */
0x41, /* LATIN CAPITAL LETTER A */
0xCC, 0x82, /* COMBINING CIRCUMFLEX ACCENT */
0x41, /* LATIN CAPITAL LETTER A */
0xCC, 0x83, /* COMBINING TILDE */
0x41, /* LATIN CAPITAL LETTER A */
0xCC, 0x88, /* COMBINING DIAERESIS */
0x41, /* LATIN CAPITAL LETTER A */
0xCC, 0x8A, /* COMBINING RING ABOVE */
0x43, /* LATIN CAPITAL LETTER C */
0xCC, 0xA7, /* COMBINING CEDILLA */
0x45, /* LATIN CAPITAL LETTER E */
0xCC, 0x80, /* COMBINING GRAVE ACCENT */
0x45, /* LATIN CAPITAL LETTER E */
0xCC, 0x81, /* COMBINING ACUTE ACCENT */
0x45, /* LATIN CAPITAL LETTER E */
0xCC, 0x82, /* COMBINING CIRCUMFLEX ACCENT */
0x45, /* LATIN CAPITAL LETTER E */
0xCC, 0x88, /* COMBINING DIAERESIS */
0x49, /* LATIN CAPITAL LETTER I */
0xCC, 0x80, /* COMBINING GRAVE ACCENT */
0x49, /* LATIN CAPITAL LETTER I */
0xCC, 0x81, /* COMBINING ACUTE ACCENT */
0x49, /* LATIN CAPITAL LETTER I */
0xCC, 0x82, /* COMBINING CIRCUMFLEX ACCENT */
0x49, /* LATIN CAPITAL LETTER I */
0xCC, 0x88, /* COMBINING DIAERESIS */
0x4E, /* LATIN CAPITAL LETTER N */
0xCC, 0x83, /* COMBINING TILDE */
0x00
};
/* NFC normalized UTF-8 encoding for same Unicode string: */
unsigned char nname_utf8[] = {
0xC3, 0x80, /* LATIN CAPITAL LETTER A WITH GRAVE */
0xC3, 0x81, /* LATIN CAPITAL LETTER A WITH ACUTE */
0xC3, 0x82, /* LATIN CAPITAL LETTER A WITH CIRCUMFLEX */
0xC3, 0x83, /* LATIN CAPITAL LETTER A WITH TILDE */
0xC3, 0x84, /* LATIN CAPITAL LETTER A WITH DIAERESIS */
0xC3, 0x85, /* LATIN CAPITAL LETTER A WITH RING ABOVE */
0xC3, 0x87, /* LATIN CAPITAL LETTER C WITH CEDILLA */
0xC3, 0x88, /* LATIN CAPITAL LETTER E WITH GRAVE */
0xC3, 0x89, /* LATIN CAPITAL LETTER E WITH ACUTE */
0xC3, 0x8A, /* LATIN CAPITAL LETTER E WITH CIRCUMFLEX */
0xC3, 0x8B, /* LATIN CAPITAL LETTER E WITH DIAERESIS */
0xC3, 0x8C, /* LATIN CAPITAL LETTER I WITH GRAVE */
0xC3, 0x8D, /* LATIN CAPITAL LETTER I WITH ACUTE */
0xC3, 0x8E, /* LATIN CAPITAL LETTER I WITH CIRCUMFLEX */
0xC3, 0x8F, /* LATIN CAPITAL LETTER I WITH DIAERESIS */
0xC3, 0x91, /* LATIN CAPITAL LETTER N WITH TILDE */
0x00
};
/* Unnormalized name used for dimension, variable, and attribute value */
#define UNAME ((char *) uname_utf8)
#define UNAMELEN (sizeof uname_utf8)
/* Normalized name */
#define NNAME ((char *) nname_utf8)
#define NNAMELEN (sizeof nname_utf8)
char name_in[UNAMELEN + 1], strings_in[UNAMELEN + 1];
nc_type att_type;
MPI_Offset att_len;
int err, dimid_in, varid_in, attnum_in;
int attvals[] = {42};
#define ATTNUM ((sizeof attvals)/(sizeof attvals[0]))
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL,&ncid); ERR
/* Define dimension with unnormalized Unicode UTF-8 encoded name */
err = ncmpi_def_dim(ncid, UNAME, NX, &dimid); ERR
dimids[0] = dimid;
/* Define variable with same name */
err = ncmpi_def_var(ncid, UNAME, NC_CHAR, NDIMS, dimids, &varid); ERR
/* Create string attribute with same value */
err = ncmpi_put_att_text(ncid, varid, UNITS, UNAMELEN, UNAME); ERR
/* Create int attribute with same name */
err = ncmpi_put_att_int(ncid, varid, UNAME, NC_INT, ATTNUM, attvals); ERR
/* Try to create dimension and variable with NFC-normalized
* version of same name. These should fail, as unnormalized name
* should have been normalized in library, so these are attempts to
* create duplicate netCDF objects. */
if ((err = ncmpi_def_dim(ncid, NNAME, NX, &dimid)) != NC_ENAMEINUSE) {
printf("Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err);
return 1;
}
if ((err=ncmpi_def_var(ncid, NNAME, NC_CHAR, NDIMS, dimids, &varid)) != NC_ENAMEINUSE) {
printf("Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err);
return 1;
}
err = ncmpi_enddef(ncid); ERR
/* Write string data, UTF-8 encoded, to the file */
err = ncmpi_put_var_text_all(ncid, varid, UNAME); ERR
err = ncmpi_close(ncid); ERR
/* Check it out. */
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR
err = ncmpi_inq_varid(ncid, UNAME, &varid); ERR
err = ncmpi_inq_varname(ncid, varid, name_in); ERR
err = strncmp(NNAME, name_in, NNAMELEN); ERR
err = ncmpi_inq_varid(ncid, NNAME, &varid_in); ERR
if ((err = ncmpi_inq_dimid(ncid, UNAME, &dimid_in)) || dimid != dimid_in)
{printf("Error at line %d\n",__LINE__);return 1;}
if ((err = ncmpi_inq_dimid(ncid, NNAME, &dimid_in)) || dimid != dimid_in)
{printf("Error at line %d\n",__LINE__);return 1;}
err = ncmpi_inq_att(ncid, varid, UNITS, &att_type, &att_len); ERR
if ( att_type != NC_CHAR || att_len != UNAMELEN)
{printf("Error at line %d\n",__LINE__);return 1;}
err = ncmpi_get_att_text(ncid, varid, UNITS, strings_in); ERR
strings_in[UNAMELEN] = '\0';
err = strncmp(UNAME, strings_in, UNAMELEN); ERR
if ((err = ncmpi_inq_attid(ncid, varid, UNAME, &attnum_in)) || ATTNUM != attnum_in)
{printf("Error at line %d\n",__LINE__);return 1;}
if ((err = ncmpi_inq_attid(ncid, varid, NNAME, &attnum_in)) || ATTNUM != attnum_in)
{printf("Error at line %d\n",__LINE__);return 1;}
err = ncmpi_close(ncid); ERR
return 0;
}
int main(int argc, char **argv) {
char dir_name[256], filename[256];
int err, rank, nerrs=0, format, ncid;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (argc != 2) {
if (!rank) printf("Usage: %s dir_name\n",argv[0]);
MPI_Finalize();
return 0;
}
strcpy(dir_name, argv[1]);
MPI_Bcast(dir_name, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
if (rank == 0) {
char cmd_str[256];
sprintf(cmd_str, "*** TESTING C %s for inquiring CDF file formats ", argv[0]);
printf("%-66s ------ ", cmd_str);
}
/* test CDF-1 -----------------------------------------------------------*/
sprintf(filename,"%s/test_cdf1.nc",dir_name);
err = ncmpi_open(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid); ERR
err = ncmpi_inq_format(ncid, &format); ERR
if (format != NC_FORMAT_CLASSIC) {
printf("Error (line=%d): expecting CDF-1 format for file %s but got %d\n",
__LINE__,filename,format);
nerrs++;
}
err = ncmpi_close(ncid); ERR
err = ncmpi_inq_file_format(filename, &format); ERR
if (format != NC_FORMAT_CLASSIC) {
printf("Error (line=%d): expecting CDF-1 format for file %s but got %d\n",
__LINE__,filename,format);
nerrs++;
}
/* test CDF-2 -----------------------------------------------------------*/
sprintf(filename,"%s/test_cdf2.nc",dir_name);
err = ncmpi_open(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid); ERR
err = ncmpi_inq_format(ncid, &format); ERR
if (format != NC_FORMAT_CDF2) {
printf("Error (line=%d): expecting CDF-2 format for file %s but got %d\n",
__LINE__,filename,format);
nerrs++;
}
err = ncmpi_close(ncid); ERR
err = ncmpi_inq_file_format(filename, &format); ERR
if (format != NC_FORMAT_CDF2) {
printf("Error (line=%d): expecting CDF-2 format for file %s but got %d\n",
__LINE__,filename,format);
nerrs++;
}
/* test CDF-5 -----------------------------------------------------------*/
sprintf(filename,"%s/test_cdf5.nc",dir_name);
err = ncmpi_open(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid); ERR
err = ncmpi_inq_format(ncid, &format); ERR
if (format != NC_FORMAT_CDF5) {
printf("Error (line=%d): expecting CDF-5 format for file %s but got %d\n",
__LINE__,filename,format);
nerrs++;
}
err = ncmpi_close(ncid); ERR
err = ncmpi_inq_file_format(filename, &format); ERR
if (format != NC_FORMAT_CDF5) {
printf("Error (line=%d): expecting CDF-5 format for file %s but got %d\n",
__LINE__,filename,format);
nerrs++;
}
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return 0;
}
/*----< main() >------------------------------------------------------------*/
int main(int argc, char **argv)
{
int i, j, err, nerrs=0, rank, nprocs;
int ncid, dimid[2], varid, req, status;
MPI_Offset start[2], count[2], stride[2], imap[2];
int var[6][4];
float k, rh[4][6];
signed char varT[4][6];
char filename[256];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for get/put varm ", basename(argv[0]));
printf("%-66s ------ ", cmd_str); fflush(stdout);
free(cmd_str);
}
#ifdef DEBUG
if (nprocs > 1 && rank == 0)
printf("Warning: %s is designed to run on 1 process\n", argv[0]);
#endif
err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_DATA,
MPI_INFO_NULL, &ncid); CHECK_ERR
/* define a variable of a 6 x 4 integer array in the nc file */
err = ncmpi_def_dim(ncid, "Y", 6, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 4, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid); CHECK_ERR
err = ncmpi_enddef(ncid); CHECK_ERR
/* create a 6 x 4 integer variable in the file with contents:
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
16, 17, 18, 19,
20, 21, 22, 23
*/
for (j=0; j<6; j++) for (i=0; i<4; i++) var[j][i] = j*4+i;
start[0] = 0; start[1] = 0;
count[0] = 6; count[1] = 4;
if (rank > 0) count[0] = count[1] = 0;
err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR
if (nprocs > 1) MPI_Barrier(MPI_COMM_WORLD);
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR
/* read the variable back in the matrix transposed way, rh is 4 x 6 */
count[0] = 6; count[1] = 4;
stride[0] = 1; stride[1] = 1;
imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */
for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0;
err = ncmpi_iget_varm_float(ncid, varid, start, count, stride, imap, &rh[0][0], &req); CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR
err = status; CHECK_ERR
/* check the contents of read */
k = 0.0;
for (i=0; i<6; i++) {
for (j=0; j<4; j++) {
if (rh[j][i] != k) {
#ifdef PRINT_ERR_ON_SCREEN
printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n",
__LINE__,__FILE__,j,i,k,rh[j][i]);
#endif
nerrs++;
break;
}
k += 1.0;
}
}
#ifdef PRINT_ON_SCREEN
/* print the contents of read */
for (j=0; j<4; j++) {
printf("[%2d]: ",j);
for (i=0; i<6; i++) {
printf("%5.1f",rh[j][i]);
}
printf("\n");
}
#endif
/* the stdout should be:
[ 0]: 0.0 4.0 8.0 12.0 16.0 20.0
[ 1]: 1.0 5.0 9.0 13.0 17.0 21.0
[ 2]: 2.0 6.0 10.0 14.0 18.0 22.0
[ 3]: 3.0 7.0 11.0 15.0 19.0 23.0
*/
for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0;
err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, &rh[0][0]); CHECK_ERR
/* check the contents of read */
k = 0.0;
for (i=0; i<6; i++) {
for (j=0; j<4; j++) {
if (rh[j][i] != k) {
#ifdef PRINT_ERR_ON_SCREEN
printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n",
__LINE__,__FILE__,j,i,k,rh[j][i]);
#endif
nerrs++;
break;
}
k += 1.0;
}
}
#ifdef PRINT_ON_SCREEN
/* print the contents of read */
for (j=0; j<4; j++) {
printf("[%2d]: ",j);
for (i=0; i<6; i++) {
printf("%5.1f",rh[j][i]);
}
printf("\n");
}
#endif
/* the stdout should be:
[ 0]: 0.0 4.0 8.0 12.0 16.0 20.0
[ 1]: 1.0 5.0 9.0 13.0 17.0 21.0
[ 2]: 2.0 6.0 10.0 14.0 18.0 22.0
[ 3]: 3.0 7.0 11.0 15.0 19.0 23.0
*/
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR
/* testing get_varm(), first zero-out the variable in the file */
memset(&var[0][0], 0, 6*4*sizeof(int));
start[0] = 0; start[1] = 0;
count[0] = 6; count[1] = 4;
if (rank > 0) count[0] = count[1] = 0;
err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR
/* set the contents of the write buffer varT, a 4 x 6 char array
50, 51, 52, 53, 54, 55,
56, 57, 58, 59, 60, 61,
62, 63, 64, 65, 66, 67,
68, 69, 70, 71, 72, 73
*/
for (j=0; j<4; j++) for (i=0; i<6; i++) varT[j][i] = j*6+i + 50;
/* write varT to the NC variable in the matrix transposed way */
start[0] = 0; start[1] = 0;
count[0] = 6; count[1] = 4;
stride[0] = 1; stride[1] = 1;
imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */
if (rank > 0) count[0] = count[1] = 0;
err = ncmpi_iput_varm_schar(ncid, varid, start, count, stride, imap, &varT[0][0], &req); CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR
err = status; CHECK_ERR
/* the output from command "ncmpidump -v var test.nc" should be:
var =
50, 56, 62, 68,
51, 57, 63, 69,
52, 58, 64, 70,
53, 59, 65, 71,
54, 60, 66, 72,
55, 61, 67, 73 ;
*/
/* check if the contents of write buffer have been altered */
for (j=0; j<4; j++) {
for (i=0; i<6; i++) {
if (varT[j][i] != j*6+i + 50) {
#ifdef PRINT_ERR_ON_SCREEN
/* this error is a pnetcdf internal error, if occurs */
printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n",
__LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]);
#endif
nerrs++;
break;
}
}
}
err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, imap, &varT[0][0]); CHECK_ERR
/* check if the contents of write buffer have been altered */
for (j=0; j<4; j++) {
for (i=0; i<6; i++) {
if (varT[j][i] != j*6+i + 50) {
#ifdef PRINT_ERR_ON_SCREEN
/* this error is a pnetcdf internal error, if occurs */
printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n",
__LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]);
#endif
nerrs++;
break;
}
}
}
err = ncmpi_close(ncid); CHECK_ERR
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
if (malloc_size > 0) ncmpi_inq_malloc_list();
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
/*
* NETCDF_open()
* Open the trajectory specified by the filename and accessMode in trajInfo as
* a NETCDF traj.
* Return 0 on success, 1 on failure
*/
int NETCDF_open(coordinateInfo *trajInfo) {
#ifdef BINTRAJ
int err,ncid;
// NOTE: Put in a check, only open if coord is unknown or netcdf
if (prnlev>0) fprintf(stdout,"[%i] NETCDF_open(): Opening %s\n",
worldrank,trajInfo->filename);
switch (trajInfo->accessMode) {
case 0: // Read
# ifdef MPI
err = ncmpi_open(MPI_COMM_WORLD, trajInfo->filename, NC_NOWRITE, MPI_INFO_NULL, &ncid);
/* This next line is a test. Apparently it puts the netcdf file in an
* independent I/O mode. Not sure if it is bad to always put here.
* Originally this call was only made from ptrajPreprocess...
*/
if (err == NC_NOERR)
err = ncmpi_begin_indep_data(ncid);
# else
err = nc_open(trajInfo->filename, NC_NOWRITE, &ncid);
# endif
break;
case 1: // Write
//omode=NC_WRITE;
# ifdef MPI
err = ncmpi_create(MPI_COMM_WORLD, trajInfo->filename, NC_64BIT_OFFSET, MPI_INFO_NULL, &ncid);
if (err == NC_NOERR)
ncmpi_begin_indep_data(ncid);
# else
err = nc_create(trajInfo->filename, NC_64BIT_OFFSET, &ncid);
# endif
break;
case 2: // Append
printfone("Appending of NETCDF files is not supported.\n");
return 1;
break;
}
/* If opening succeeded and memory hasnt been allocated already
* initialize necessary data structure.
* NOTE: Should this be in NETCDF_setup? If so ncid would have to
* be its own variable in coordinateInfo.
* NOTE: If this is an output file trajInfo->type has already been set.
* Not a huge problem but is a bit circular. TRAJOUT should eventually
* only set trajInfo->isNetcdf.
*/
if (err == NC_NOERR) {
trajInfo->type = COORD_AMBER_NETCDF;
if (trajInfo->NCInfo==NULL) {
trajInfo->NCInfo = (netcdfTrajectoryInfo *) safe_malloc(sizeof(netcdfTrajectoryInfo));
INITIALIZE_netcdfTrajectoryInfo( trajInfo->NCInfo );
}
trajInfo->NCInfo->currentFrame = worldrank;
// Always set NCID since it can change depending on when file is opened
trajInfo->NCInfo->ncid = ncid;
if (prnlev>0) fprintf(stdout,"NETCDF_open(): %s has been assigned ncid of %i\n",
trajInfo->filename,ncid);
return 0;
}
// If we are here an error occured. Print the error message before exiting.
fprintf(stdout,"Error: NETCDF_open(): Could not open %s with accessMode %i\n",
trajInfo->filename,trajInfo->accessMode);
fprintf(stdout,"%s\n",nc_strerror(err));
#endif
// If no BINTRAJ always fail
return 1;
}
int
main(int argc, char **argv) {
int stat; /* return status */
int ncid; /* netCDF id */
int rec, i, j, k;
signed char x[] = {42, 21};
/* dimension ids */
int rec_dim;
int i_dim;
int j_dim;
int k_dim;
int n_dim;
#define NUMRECS 1
#define I_LEN 4104
#define J_LEN 1023
#define K_LEN 1023
#define N_LEN 2
/* dimension lengths */
MPI_Offset rec_len = NC_UNLIMITED;
MPI_Offset i_len = I_LEN;
MPI_Offset j_len = J_LEN;
MPI_Offset k_len = K_LEN;
MPI_Offset n_len = N_LEN;
/* variable ids */
int var1_id;
int x_id;
/* rank (number of dimensions) for each variable */
# define RANK_var1 4
# define RANK_x 2
/* variable shapes */
int var1_dims[RANK_var1];
int x_dims[RANK_x];
printf("\n*** Testing large files, slowly.\n");
printf("*** Creating large file %s...", FILE_NAME);
MPI_Init(&argc, &argv);
/* enter define mode */
stat = ncmpi_create(MPI_COMM_WORLD, FILE_NAME, NC_CLOBBER|NC_64BIT_OFFSET,
MPI_INFO_NULL, &ncid);
check_err(stat,__LINE__,__FILE__);
/* define dimensions */
stat = ncmpi_def_dim(ncid, "rec", rec_len, &rec_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "i", i_len, &i_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "j", j_len, &j_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "k", k_len, &k_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "n", n_len, &n_dim);
check_err(stat,__LINE__,__FILE__);
/* define variables */
var1_dims[0] = rec_dim;
var1_dims[1] = i_dim;
var1_dims[2] = j_dim;
var1_dims[3] = k_dim;
stat = ncmpi_def_var(ncid, "var1", NC_BYTE, RANK_var1, var1_dims, &var1_id);
check_err(stat,__LINE__,__FILE__);
x_dims[0] = rec_dim;
x_dims[1] = n_dim;
stat = ncmpi_def_var(ncid, "x", NC_BYTE, RANK_x, x_dims, &x_id);
check_err(stat,__LINE__,__FILE__);
/* don't initialize variables with fill values */
stat = ncmpi_set_fill(ncid, NC_NOFILL, 0);
check_err(stat,__LINE__,__FILE__);
/* leave define mode */
stat = ncmpi_enddef (ncid);
check_err(stat,__LINE__,__FILE__);
{ /* store var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0};
static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static MPI_Offset x_start[RANK_x] = {0, 0};
static MPI_Offset x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
var1[j][k] = n++;
}
}
var1_start[1] = i;
stat = ncmpi_put_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
}
}
stat = ncmpi_put_vara_schar_all(ncid, x_id, x_start, x_count, x);
check_err(stat,__LINE__,__FILE__);
}
stat = ncmpi_close(ncid);
check_err(stat,__LINE__,__FILE__);
printf("ok\n");
printf("*** Reading large file %s...", FILE_NAME);
stat = ncmpi_open(MPI_COMM_WORLD, FILE_NAME, NC_NOWRITE,
MPI_INFO_NULL, &ncid);
check_err(stat,__LINE__,__FILE__);
{ /* read var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0};
static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static MPI_Offset x_start[RANK_x] = {0, 0};
static MPI_Offset x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
var1_start[1] = i;
stat = ncmpi_get_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
if (var1[j][k] != (signed char) n) {
printf("Error on read, var1[%d, %d, %d, %d] = %d wrong, "
"should be %d !\n", rec, i, j, k, var1[j][k], (signed char) n);
return 1;
}
n++;
}
}
}
ncmpi_get_vara_schar_all(ncid, x_id, x_start, x_count, x);
if(x[0] != 42 || x[1] != 21) {
printf("Error on read, x[] = %d, %d\n", x[0], x[1]);
return 1;
}
}
}
stat = ncmpi_close(ncid);
check_err(stat,__LINE__,__FILE__);
printf("ok\n");
printf("*** Tests successful!\n");
/* Delete the file. */
(void) remove(FILE_NAME);
MPI_Finalize();
return 0;
}
void read_pnetcdf(const std::string &filename,
int iVar,
ConfigMap &configMap,
HostArray<double> &localData)
{
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
int nbMpiProc;
MPI_Comm_size(MPI_COMM_WORLD, &nbMpiProc);
// netcdf file id
int ncFileId;
int err;
// ghostWidth
int ghostWidth = configMap.getInteger("mesh","ghostWidth",3);
// file creation mode
int ncOpenMode = NC_NOWRITE;
int varIds[8];
MPI_Offset starts[3], counts[3]; // read_size, sum_read_size;
MPI_Info mpi_info_used;
// domain local size
int nx,ny,nz;
// sizes to read
//int nx_r, ny_r, nz_r; // logical sizes / per sub-domain
//int nx_g, ny_g, nz_g; // sizes with ghost zones included / per sub-domain
/* read domain sizes */
nx=configMap.getInteger("mesh","nx",32);
ny=configMap.getInteger("mesh","ny",32);
nz=configMap.getInteger("mesh","nz",32);
// nx_g = nx+2*ghostWidth;
// ny_g = ny+2*ghostWidth;
// nz_g = nz+2*ghostWidth;
// get input filename from configMap
//std::string filename = configMap.getString("input", "filename", "");
/*
* Open NetCDF file
*/
err = ncmpi_open(MPI_COMM_WORLD, filename.c_str(),
ncOpenMode,
MPI_INFO_NULL, &ncFileId);
if (err != NC_NOERR) {
printf("Error: ncmpi_open() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err));
MPI_Abort(MPI_COMM_WORLD, -1);
exit(1);
}
/*
* Query NetCDF mode
*/
int NC_mode;
err = ncmpi_inq_version(ncFileId, &NC_mode);
if (myRank==0) {
if (NC_mode == NC_64BIT_DATA)
std::cout << "Pnetcdf Input mode : NC_64BIT_DATA (CDF-5)\n";
else if (NC_mode == NC_64BIT_OFFSET)
std::cout << "Pnetcdf Input mode : NC_64BIT_OFFSET (CDF-2)\n";
else
std::cout << "Pnetcdf Input mode : unknown\n";
}
/*
* Query information about variables
*/
{
int ndims, nvars, ngatts, unlimited;
err = ncmpi_inq(ncFileId, &ndims, &nvars, &ngatts, &unlimited);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho", &varIds[ID]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "E", &varIds[IP]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vx", &varIds[IU]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vy", &varIds[IV]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vz", &varIds[IW]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "Bx", &varIds[IA]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "By", &varIds[IB]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "Bz", &varIds[IC]);
PNETCDF_HANDLE_ERROR;
} // end query information
/*
* Define expected data types (no conversion done here)
*/
//nc_type ncDataType;
MPI_Datatype mpiDataType;
//ncDataType = NC_DOUBLE;
mpiDataType = MPI_DOUBLE;
/*
* Get all the MPI_IO hints used (just in case, we want to print it after
* reading data...
*/
err = ncmpi_get_file_info(ncFileId, &mpi_info_used);
PNETCDF_HANDLE_ERROR;
/*
* Read heavy data (take care of row-major / column major format !)
*/
// use overlapping domains
// counts[IZ] = nx_rg;
// counts[IY] = ny_rg;
// counts[IX] = nz_rg;
// starts[IZ] = 0;
// starts[IY] = 0;
// starts[IX] = myRank*nz_r;
counts[IZ] = nx;
counts[IY] = ny;
counts[IX] = nz;
starts[IZ] = ghostWidth;
starts[IY] = ghostWidth;
starts[IX] = ghostWidth+myRank*nz;
int nItems = counts[IX]*counts[IY]*counts[IZ];
/*
* Actual reading
*/
{
double* data;
//data = &(localData(0,0,0,0));
data = localData.data();
err = ncmpi_get_vara_all(ncFileId, varIds[iVar],
starts, counts, data, nItems, mpiDataType);
PNETCDF_HANDLE_ERROR;
} // end for loop reading heavy data
/*
* close the file
*/
err = ncmpi_close(ncFileId);
PNETCDF_HANDLE_ERROR;
} // read_pnetcdf
//----------------------------------------------------------------
int main(int argc, char **argv)
{
init_mpi(argc, argv);
printf("start %d %d\n",rank,process_count);
int i;
if(rank==0) {
parse_args(argc, argv);
// var_count = read_list_in_file(var_file, &netcdf_var_names);
// rank_0_print("Number of variables = %d\n", var_count);
// time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
// rank_0_print("Number of timesteps = %d\n", time_step_count);
check_args();
}
// The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
MPI_Bcast(global_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(local_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
// MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&var_file, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&netcdf_file_list, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
// MPI_Bcast(&var_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&output_file_name, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
#endif
//TODO: Only the rank 0 should read the input files and broadcast the data
var_count = read_list_in_file(var_file, &netcdf_var_names);
rank_0_print("Number of variables = %d\n", var_count);
time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
rank_0_print("Number of timesteps = %d\n", time_step_count);
calculate_per_process_offsets();
create_pidx_var_names();
PIDX_access pidx_access;
create_pidx_access(&pidx_access);
PIDX_time_step_caching_ON();
determine_var_types();
create_pidx_vars();
int t = 0,plist_id;
for (t = 0; t < time_step_count; ++t)
{
rank_0_print("Processing time step %d (file %s)\n", t, netcdf_file_names[t]);
PIDX_file pidx_file;
int ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, pidx_access, &pidx_file);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: Failed to create PIDX file\n");
set_pidx_params(pidx_file);
PIDX_set_current_time_step(pidx_file, t);
int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[t], NC_NOWRITE, MPI_INFO_NULL, &plist_id);
if (file_id !=0)
terminate_with_error_msg("ERROR: Failed to open file %s\n", netcdf_file_names[t]);
int v = 0;
for(v = 0; v < var_count; ++v)
{
rank_0_print("Processing variable %s\n", netcdf_var_names[v]);
var_data = malloc(var_types[v].atomic_type * var_types[v].num_values * local_box_size[0] * local_box_size[1] * local_box_size[2]);
read_var_from_netcdf(plist_id, netcdf_var_names[v], var_types[v]);
write_var_to_idx(pidx_file, pidx_var_names[v], pidx_vars[v]);
if (PIDX_flush(pidx_file) != PIDX_success)
terminate_with_error_msg("ERROR: Failed to flush variable %s, time step %d\n", pidx_var_names[v], t);
free(var_data);
}
ncmpi_close(plist_id);
PIDX_close(pidx_file);
}
PIDX_time_step_caching_OFF();
PIDX_close_access(pidx_access);
free_memory();
shutdown_mpi();
return 0;
}
/*----< main() >------------------------------------------------------------*/
int main(int argc, char **argv) {
char filename[256];
int i, j, err, ncid, varid0, varid1, varid2, dimids[2], nerrs=0;
int rank, nprocs, debug=0, blocklengths[2], **buf, *bufptr;
int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2];
MPI_Offset start[2], count[2];
MPI_Aint a0, a1, disps[2];
MPI_Datatype buftype, ghost_buftype, rec_filetype, fix_filetype;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 0;
}
strcpy(filename, "testfile.nc");
if (argc == 2) strcpy(filename, argv[1]);
MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
if (rank == 0) {
char cmd_str[256];
sprintf(cmd_str, "*** TESTING C %s for flexible put and get ", argv[0]);
printf("%-66s ------ ", cmd_str); fflush(stdout);
}
buf = (int**)malloc(NY * sizeof(int*));
buf[0] = (int*)malloc(NY * NX * sizeof(int));
for (i=1; i<NY; i++) buf[i] = buf[i-1] + NX;
/* construct various MPI derived data types */
/* construct an MPI derived data type for swapping 1st row with 2nd row */
blocklengths[0] = blocklengths[1] = NX;
MPI_Get_address(buf[1], &a0);
MPI_Get_address(buf[0], &a1);
disps[0] = 0;
disps[1] = a1 - a0;
bufptr = buf[1];
err = MPI_Type_create_hindexed(2, blocklengths, disps, MPI_INT, &buftype);
if (err != MPI_SUCCESS) printf("MPI error MPI_Type_create_hindexed\n");
MPI_Type_commit(&buftype);
start[0] = 0; start[1] = NX*rank;
count[0] = 2; count[1] = NX;
if (debug) printf("put start=%lld %lld count=%lld %lld\n",start[0],start[1],count[0],count[1]);
/* create a file type for the fixed-size variable */
array_of_sizes[0] = 2;
array_of_sizes[1] = NX*nprocs;
array_of_subsizes[0] = count[0];
array_of_subsizes[1] = count[1];
array_of_starts[0] = start[0];
array_of_starts[1] = start[1];
MPI_Type_create_subarray(2, array_of_sizes, array_of_subsizes,
array_of_starts, MPI_ORDER_C,
MPI_INT, &fix_filetype);
MPI_Type_commit(&fix_filetype);
/* create a buftype with ghost cells on each side */
array_of_sizes[0] = count[0]+4;
array_of_sizes[1] = count[1]+4;
array_of_subsizes[0] = count[0];
array_of_subsizes[1] = count[1];
array_of_starts[0] = 2;
array_of_starts[1] = 2;
MPI_Type_create_subarray(2, array_of_sizes, array_of_subsizes,
array_of_starts, MPI_ORDER_C,
MPI_INT, &ghost_buftype);
MPI_Type_commit(&ghost_buftype);
/* create a new file for write */
err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL,
&ncid); ERR
/* define a 2D array */
err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimids[0]); ERR
err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimids[1]); ERR
err = ncmpi_def_var(ncid, "rec_var", NC_INT, 2, dimids, &varid0); ERR
err = ncmpi_def_var(ncid, "dummy_rec", NC_INT, 2, dimids, &varid2); ERR
err = ncmpi_def_dim(ncid, "FIX_DIM", 2, &dimids[0]); ERR
err = ncmpi_def_var(ncid, "fix_var", NC_INT, 2, dimids, &varid1); ERR
err = ncmpi_enddef(ncid); ERR
/* create a file type for the record variable */
int *array_of_blocklengths=(int*) malloc(count[0]*sizeof(int));
MPI_Aint *array_of_displacements=(MPI_Aint*) malloc(count[0]*sizeof(MPI_Aint));
MPI_Offset recsize;
err = ncmpi_inq_recsize(ncid, &recsize);
for (i=0; i<count[0]; i++) {
array_of_blocklengths[i] = count[1];
array_of_displacements[i] = start[1]*sizeof(int) + recsize * i;
}
MPI_Type_create_hindexed(2, array_of_blocklengths, array_of_displacements,
MPI_INT, &rec_filetype);
MPI_Type_commit(&rec_filetype);
free(array_of_blocklengths);
free(array_of_displacements);
/* initialize the contents of the array */
for (j=0; j<NY; j++) for (i=0; i<NX; i++) buf[j][i] = rank*100 + j*10 + i;
/* write the record variable */
err = ncmpi_put_vard_all(ncid, varid0, rec_filetype, bufptr, 1, buftype); ERR
/* check if the contents of buf are altered */
CHECK_VALUE
/* check if root process can write to file header in data mode */
err = ncmpi_rename_var(ncid, varid0, "rec_VAR"); ERR
/* write the fixed-size variable */
err = ncmpi_put_vard_all(ncid, varid1, fix_filetype, bufptr, 1, buftype); ERR
/* check if the contents of buf are altered */
CHECK_VALUE
/* check if root process can write to file header in data mode */
err = ncmpi_rename_var(ncid, varid0, "rec_var"); ERR
/* test the same routines in independent data mode */
err = ncmpi_begin_indep_data(ncid); ERR
err = ncmpi_put_vard(ncid, varid0, rec_filetype, bufptr, 1, buftype); ERR
CHECK_VALUE
err = ncmpi_rename_var(ncid, varid0, "rec_VAR"); ERR
err = ncmpi_put_vard(ncid, varid1, fix_filetype, bufptr, 1, buftype); ERR
CHECK_VALUE
err = ncmpi_rename_var(ncid, varid0, "rec_var"); ERR
err = ncmpi_end_indep_data(ncid); ERR
err = ncmpi_close(ncid); ERR
/* open the same file and read back for validate */
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL,
&ncid); ERR
err = ncmpi_inq_varid(ncid, "rec_var", &varid0); ERR
err = ncmpi_inq_varid(ncid, "fix_var", &varid1); ERR
nerrs += get_var_and_verify(ncid, varid0, start, count, buf, buftype, ghost_buftype, rec_filetype);
nerrs += get_var_and_verify(ncid, varid1, start, count, buf, buftype, ghost_buftype, fix_filetype);
err = ncmpi_close(ncid); ERR
MPI_Type_free(&rec_filetype);
MPI_Type_free(&fix_filetype);
MPI_Type_free(&buftype);
MPI_Type_free(&ghost_buftype);
free(buf[0]); free(buf);
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return 0;
}
int
main(int argc, char **argv)
{
int rank, nprocs, ncid, pres_varid, temp_varid;
int lat_varid, lon_varid;
/* The start and count arrays will tell the netCDF library where to
read our data. */
MPI_Offset start[NDIMS], count[NDIMS];
/* Program variables to hold the data we will read. We will only
need enough space to hold one timestep of data; one record. */
float **pres_in=NULL; /* [NLVL/nprocs][NLAT][NLON] */
float **temp_in=NULL; /* [NLVL/nprocs][NLAT][NLON] */
/* These program variables hold the latitudes and longitudes. */
float lats[NLAT], lons[NLON];
/* Loop indexes. */
int lvl, lat, lon, rec, i = 0;
/* Error handling. */
int err, nerrs=0;
char *filename = FILE_NAME;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) filename = argv[1];
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for reading file", basename(argv[0]));
printf("%-66s ------ ", cmd_str);
free(cmd_str);
}
/* Open the file. */
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid);
CHECK_ERR
/* Get the varids of the latitude and longitude coordinate
* variables. */
err = ncmpi_inq_varid(ncid, LAT_NAME, &lat_varid);
CHECK_ERR
err = ncmpi_inq_varid(ncid, LON_NAME, &lon_varid);
CHECK_ERR
/* Read the coordinate variable data. */
memset(lats, 0, sizeof(float)*NLAT);
memset(lons, 0, sizeof(float)*NLON);
err = ncmpi_get_var_float_all(ncid, lat_varid, &lats[0]);
CHECK_ERR
err = ncmpi_get_var_float_all(ncid, lon_varid, &lons[0]);
CHECK_ERR
/* Check the coordinate variable data. */
for (lat = 0; lat < NLAT; lat++)
if (lats[lat] != START_LAT + 5.*lat) {
printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LAT+5.*lat,lats[lat]);
nerrs++;
goto fn_exit;
}
for (lon = 0; lon < NLON; lon++)
if (lons[lon] != START_LON + 5.*lon) {
printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LON+5.*lon,lons[lon]);
nerrs++;
goto fn_exit;
}
/* Get the varids of the pressure and temperature netCDF
* variables. */
err = ncmpi_inq_varid(ncid, PRES_NAME, &pres_varid);
CHECK_ERR
err = ncmpi_inq_varid(ncid, TEMP_NAME, &temp_varid);
CHECK_ERR
/* Read the data. Since we know the contents of the file we know
* that the data arrays in this program are the correct size to
* hold one timestep. */
count[0] = 1;
count[2] = NLAT;
count[3] = NLON;
start[2] = 0;
start[3] = 0;
/* divide NLVL dimension among processes */
count[1] = NLVL / nprocs;
start[1] = count[1] * rank;
if (rank < NLVL % nprocs) {
start[1] += rank;
count[1]++;
}
else {
start[1] += NLVL % nprocs;
}
if (count[1] == 0) start[1] = 0;
/* allocate read buffers */
pres_in = (float**) malloc(count[1]*2 * sizeof(float*));
temp_in = pres_in + count[1];
if (count[1] > 0) {
pres_in[0] = (float*) malloc(count[1]*2 * NLAT*NLON * sizeof(float));
temp_in[0] = pres_in[0] + count[1] * NLAT*NLON;
for (i=1; i<count[1]; i++) {
pres_in[i] = pres_in[i-1] + NLAT*NLON;
temp_in[i] = temp_in[i-1] + NLAT*NLON;
}
}
/* Read and check one record at a time. */
for (rec = 0; rec < NREC; rec++)
{
start[0] = rec;
err = ncmpi_get_vara_float_all(ncid, pres_varid, start, count, &pres_in[0][0]);
CHECK_ERR
err = ncmpi_get_vara_float_all(ncid, temp_varid, start, count, &temp_in[0][0]);
CHECK_ERR
/* Check the data. */
i = (int)start[1] * NLAT * NLON;
for (lvl=0; lvl<count[1]; lvl++)
for (lat = 0; lat < NLAT; lat++)
for (lon = 0; lon < NLON; lon++) {
if (pres_in[lvl][lat*NLON+lon] != SAMPLE_PRESSURE + i) {
printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_PRESSURE+i,pres_in[lvl][lat*NLON+lon]);
nerrs++;
goto fn_exit;
}
if (temp_in[lvl][lat*NLON+lon] != SAMPLE_TEMP + i) {
printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_TEMP+i,temp_in[lvl][lat*NLON+lon]);
nerrs++;
goto fn_exit;
}
i++;
}
} /* next record */
fn_exit:
/* Close the file. */
err = ncmpi_close(ncid);
CHECK_ERR
if (pres_in != NULL) {
if (pres_in[0] != NULL) free(pres_in[0]);
free(pres_in);
}
/* check if there is any malloc residue */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
int main(int argc, char **argv) {
int i, j, rank, nprocs, ret;
int ncfile, ndims, nvars, ngatts, unlimited;
int var_ndims, var_natts;;
MPI_Offset *dim_sizes, var_size;
MPI_Offset *start, *count;
int *requests, *statuses;
char varname[NC_MAX_NAME+1];
int dimids[NC_MAX_VAR_DIMS];
nc_type type;
int **data;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc != 2) {
if (rank == 0) printf("Usage: %s filename\n", argv[0]);
MPI_Finalize();
exit(-1);
}
ret = ncmpi_open(MPI_COMM_WORLD, argv[1], NC_NOWRITE, MPI_INFO_NULL,
&ncfile);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* reader knows nothing about dataset, but we can interrogate with query
* routines: ncmpi_inq tells us how many of each kind of "thing"
* (dimension, variable, attribute) we will find in the file */
/* no commnunication needed after ncmpi_open: all processors have a cached
* veiw of the metadata once ncmpi_open returns */
ret = ncmpi_inq(ncfile, &ndims, &nvars, &ngatts, &unlimited);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* we do not really need the name of the dimension or the variable for
* reading in this example. we could, in a different example, take the
* name of a variable on the command line and read just that one */
dim_sizes = calloc(ndims, sizeof(MPI_Offset));
/* netcdf dimension identifiers are allocated sequentially starting
* at zero; same for variable identifiers */
for(i=0; i<ndims; i++) {
ret = ncmpi_inq_dimlen(ncfile, i, &(dim_sizes[i]) );
if (ret != NC_NOERR) handle_error(ret, __LINE__);
}
requests = calloc(nvars, sizeof(int));
statuses = calloc(nvars, sizeof(int));
data = malloc(nvars*sizeof(int*));
for(i=0; i<nvars; i++) {
/* much less coordination in this case compared to rank 0 doing all
* the i/o: everyone already has the necessary information */
ret = ncmpi_inq_var(ncfile, i, varname, &type, &var_ndims, dimids,
&var_natts);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
start = calloc(var_ndims, sizeof(MPI_Offset));
count = calloc(var_ndims, sizeof(MPI_Offset));
/* we will simply decompose along one dimension. Generally the
* application has some algorithim for domain decomposistion. Note
* that data decomposistion can have an impact on i/o performance.
* Often it's best just to do what is natural for the application,
* but something to consider if performance is not what was
* expected/desired */
start[0] = (dim_sizes[dimids[0]]/nprocs)*rank;
count[0] = (dim_sizes[dimids[0]]/nprocs);
var_size = count[0];
for (j=1; j<var_ndims; j++) {
start[j] = 0;
count[j] = dim_sizes[dimids[j]];
var_size *= count[j];
}
switch(type) {
case NC_INT:
data[i] = calloc(var_size, sizeof(int));
/* as with the writes, this call is independent: we
* will do any coordination (if desired) in a
* subsequent ncmpi_wait_all() call */
ret = ncmpi_iget_vara(ncfile, i, start, count, data[i],
var_size, MPI_INT, &(requests[i]));
if (ret != NC_NOERR) handle_error(ret, __LINE__);
break;
default:
/* we can do this for all the known netcdf types but this
* example is already getting too long */
fprintf(stderr, "unsupported NetCDF type \n");
}
free(start);
free(count);
}
ret = ncmpi_wait_all(ncfile, nvars, requests, statuses);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* now that the ncmpi_wait_all has returned, the caller can do stuff with
* the buffers passed in to the non-blocking operations. The buffer resue
* rules are similar to MPI non-blocking messages */
for (i=0; i<nvars; i++) {
if (data[i] != NULL) free(data[i]);
}
free(data);
free(dim_sizes);
free(requests);
free(statuses);
ret = ncmpi_close(ncfile);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
MPI_Finalize();
return 0;
}
/*
reads input in pnetcdf format
nblocks: (output) local number of blocks
tot_blocks: (output) total number of blocks
vblocks: (output) pointer to array of vblocks
in_file: input file name
comm: MPI communicator
gids: (output) gids of local blocks (allocated by this function)
num_neighbors: (output) number of neighbors for each local block
(allocated by this function)
neighbors: (output) gids of neighbors of each local block
(allocated by this function)
side effects: allocates vblocks, gids, num_neighbors, neighbors
*/
void pnetcdf_read(int *nblocks, int *tot_blocks, struct vblock_t ***vblocks,
char *in_file, MPI_Comm comm, int *gids, int *num_neighbors,
int **neighbors) {
#ifdef USEPNETCDF
int err;
int ncid, varids[23], dimids[8];
MPI_Offset start[2], count[2];
nc_type type;
int ndims, natts;
int dims[2];
int rank, groupsize; /* MPI usual */
/* open file for reading */
err = ncmpi_open(comm, in_file, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR;
err = ncmpi_inq_varid(ncid, "block_off_num_verts", &varids[5]); ERR;
err = ncmpi_inq_varid(ncid, "block_off_num_complete_cells", &varids[6]); ERR;
err = ncmpi_inq_varid(ncid, "block_off_tot_num_cell_faces", &varids[7]); ERR;
err = ncmpi_inq_varid(ncid, "block_off_tot_num_face_verts", &varids[8]); ERR;
err = ncmpi_inq_varid(ncid, "block_off_num_orig_particles", &varids[9]); ERR;
/* get number of blocks */
MPI_Offset num_g_blocks; /* 64 bit version of tot_blcoks */
err = ncmpi_inq_dimlen(ncid, dimids[0], &num_g_blocks); ERR;
*tot_blocks = num_g_blocks;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &groupsize);
int start_block_ofst = rank * (*tot_blocks / groupsize);
*nblocks = (rank < groupsize - 1 ? (*tot_blocks / groupsize) :
*tot_blocks - (rank * *tot_blocks / groupsize));
/* block offsets */
int64_t *block_ofsts = (int64_t*)malloc(*tot_blocks * sizeof(int64_t));
*vblocks = (struct vblock_t**)malloc(*nblocks * sizeof(struct vblock_t*));
/* read all blocks */
gids = (int *)malloc(*nblocks * sizeof(int));
num_neighbors = (int *)malloc(*nblocks * sizeof(int));
neighbors = (int **)malloc(*nblocks * sizeof(int *));
int b;
for (b = 0; b < *nblocks; b++) {
struct vblock_t* v = (struct vblock_t*)malloc(sizeof(struct vblock_t));
/* quantities */
start[0] = start_block_ofst + b;
count[0] = 1;
err = ncmpi_inq_varid(ncid, "num_verts", &varids[0]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[0], start, count,
&(v->num_verts)); ERR;
err = ncmpi_inq_varid(ncid, "num_complete_cells", &varids[1]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[1], start, count,
&(v->num_complete_cells)); ERR;
err = ncmpi_inq_varid(ncid, "tot_num_cell_faces", &varids[2]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[2], start, count,
&(v->tot_num_cell_faces)); ERR;
err = ncmpi_inq_varid(ncid, "tot_num_face_verts", &varids[3]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[3], start, count,
&(v->tot_num_face_verts)); ERR;
err = ncmpi_inq_varid(ncid, "num_orig_particles", &varids[4]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[4], start, count,
&(v->num_orig_particles)); ERR;
err = ncmpi_inq_varid(ncid, "neighbors", &varids[21]); ERR;
err = ncmpi_inq_var(ncid, varids[21], 0, &type, &ndims,
dimids, &natts);
/* block bounds */
start[0] = start_block_ofst + b;
start[1] = 0;
count[0] = 1;
count[1] = 3;
err = ncmpi_inq_varid(ncid, "mins", &varids[11]); ERR;
err = ncmpi_get_vara_float_all(ncid, varids[11], start, count,
v->mins); ERR;
err = ncmpi_inq_varid(ncid, "maxs", &varids[12]); ERR;
err = ncmpi_get_vara_float_all(ncid, varids[12], start, count,
v->maxs); ERR;
/* save_verts */
start[0] = 0;
count[0] = *tot_blocks;
err = ncmpi_get_vara_longlong_all(ncid, varids[5], start, count,
block_ofsts); ERR;
v->save_verts = (float *)malloc(v->num_verts * 3 * sizeof(float));
start[0] = block_ofsts[start_block_ofst + b];
start[1] = 0;
count[0] = v->num_verts;
count[1] = 3;
err = ncmpi_inq_varid(ncid, "save_verts", &varids[13]); ERR;
err = ncmpi_get_vara_float_all(ncid, varids[13], start, count,
v->save_verts); ERR;
/* sites */
start[0] = 0;
count[0] = *tot_blocks;
err = ncmpi_get_vara_longlong_all(ncid, varids[9], start, count,
block_ofsts); ERR;
v->sites = (float *)malloc(v->num_orig_particles * 3 * sizeof(float));
start[0] = block_ofsts[start_block_ofst + b];
start[1] = 0;
count[0] = v->num_orig_particles;
count[1] = 3;
err = ncmpi_inq_varid(ncid, "sites", &varids[14]); ERR;
err = ncmpi_get_vara_float_all(ncid, varids[14], start, count,
v->sites); ERR;
/* complete cells */
start[0] = 0;
count[0] = *tot_blocks;
err = ncmpi_get_vara_longlong_all(ncid, varids[6], start, count,
block_ofsts); ERR;
v->complete_cells = (int *)malloc(v->num_complete_cells * sizeof(int));
start[0] = block_ofsts[start_block_ofst + b];
count[0] = v->num_complete_cells;
err = ncmpi_inq_varid(ncid, "complete_cells", &varids[15]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[15], start, count,
v->complete_cells); ERR;
/* areas, uses same block offsets as complete cells */
v->areas = (float *)malloc(v->num_complete_cells * sizeof(float));
start[0] = block_ofsts[start_block_ofst + b];
count[0] = v->num_complete_cells;
err = ncmpi_inq_varid(ncid, "areas", &varids[16]); ERR;
err = ncmpi_get_vara_float_all(ncid, varids[16], start, count,
v->areas); ERR;
/* volumes, uses same block offsets as complete cells */
v->vols = (float *)malloc(v->num_complete_cells * sizeof(float));
start[0] = block_ofsts[start_block_ofst + b];
count[0] = v->num_complete_cells;
err = ncmpi_inq_varid(ncid, "vols", &varids[17]); ERR;
err = ncmpi_get_vara_float_all(ncid, varids[17], start, count,
v->vols); ERR;
/* num_cell_faces, uses same block offsets as complete cells */
v->num_cell_faces = (int *)malloc(v->num_complete_cells * sizeof(int));
start[0] = block_ofsts[start_block_ofst + b];
count[0] = v->num_complete_cells;
err = ncmpi_inq_varid(ncid, "num_cell_faces", &varids[18]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[18], start, count,
v->num_cell_faces); ERR;
/* num_face_verts */
start[0] = 0;
count[0] = *tot_blocks;
err = ncmpi_get_vara_longlong_all(ncid, varids[7], start, count,
block_ofsts); ERR;
v->num_face_verts = (int *)malloc(v->tot_num_cell_faces * sizeof(int));
start[0] = block_ofsts[start_block_ofst + b];
count[0] = v->tot_num_cell_faces;
err = ncmpi_inq_varid(ncid, "num_face_verts", &varids[19]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[19], start, count,
v->num_face_verts); ERR;
/* face_verts */
start[0] = 0;
count[0] = *tot_blocks;
err = ncmpi_get_vara_longlong_all(ncid, varids[8], start, count,
block_ofsts); ERR;
v->face_verts = (int *)malloc(v->tot_num_face_verts * sizeof(int));
start[0] = block_ofsts[start_block_ofst + b];
count[0] = v->tot_num_face_verts;
err = ncmpi_inq_varid(ncid, "face_verts", &varids[20]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[20], start, count,
v->face_verts); ERR;
/* neighbors */
MPI_Offset n; /* temporary 64-bit version of number of neighbors */
err = ncmpi_inq_varid(ncid, "neighbors", &varids[21]); ERR;
err = ncmpi_inq_var(ncid, varids[2], 0, &type, &ndims,
dims, &natts); ERR;
err = ncmpi_inq_dimlen(ncid, dims[0], &n); ERR;
num_neighbors[b] = n;
neighbors[b] = (int *)malloc(num_neighbors[b] * sizeof(int));
start[0] = start_block_ofst + b;
count[0] = num_neighbors[b];
err = ncmpi_get_vara_int_all(ncid, varids[21], start, count,
neighbors[b]); ERR;
/* gids */
start[0] = start_block_ofst + b;
count[0] = 1;
err = ncmpi_inq_varid(ncid, "g_block_ids", &varids[22]); ERR;
err = ncmpi_get_vara_int_all(ncid, varids[22], start, count,
&gids[b]); ERR;
(*vblocks)[b] = v;
}
/* cleanup */
err = ncmpi_close(ncid); ERR;
free(block_ofsts);
#endif
}
int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
msg = PIO_MSG_OPEN_FILE;
ios = pio_get_iosystem_from_id(iosysid);
if(ios==NULL){
printf("bad iosysid %d\n",iosysid);
return PIO_EBADID;
}
file = (file_desc_t *) malloc(sizeof(*file));
if(file==NULL){
return PIO_ENOMEM;
}
file->iotype = *iotype;
file->next = NULL;
file->iosystem = ios;
file->mode = mode;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
file->buffer.validvars=0;
file->buffer.vid=NULL;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
#ifdef _MPISERIAL
ierr = nc_open(filename, file->mode, &(file->fh));
#else
file->mode = file->mode | NC_MPIIO;
ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh));
#endif
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
// *** Note the INTENTIONAL FALLTHROUGH ***
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
// This should only be done with a file opened to append
if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){
if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT);
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
break;
}
// If we failed to open a file due to an incompatible type of NetCDF, try it
// once with just plain old basic NetCDF
#ifdef _NETCDF
if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) {
if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n");
// reset ierr on all tasks
ierr = PIO_NOERR;
// reset file markers for NETCDF on all tasks
file->iotype = PIO_IOTYPE_NETCDF;
// open netcdf file serially on main task
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh)); }
}
#endif
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr==PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
int main(int argc, char ** argv)
{
int ncid, dimid, varid;
MPI_Init(&argc, &argv);
MPI_Datatype vtype, rtype, usertype;
MPI_Aint lb, extent;
int userbufsz, *userbuf, *cmpbuf, i, errs=0;
int count = 25;
double pi = 3.14159;
MPI_Offset start, acount;
ncmpi_create(MPI_COMM_WORLD, "vectors.nc", NC_CLOBBER, MPI_INFO_NULL,
&ncid);
ncmpi_def_dim(ncid, "50k", 1024*50, &dimid);
ncmpi_def_var(ncid, "vector", NC_DOUBLE, 1, &dimid, &varid);
ncmpi_enddef(ncid);
MPI_Type_vector(VECCOUNT, BLOCKLEN, STRIDE, MPI_INT, &vtype);
MPI_Type_create_resized(vtype, 0, STRIDE*VECCOUNT*sizeof(int), &rtype);
MPI_Type_contiguous(count, rtype, &usertype);
MPI_Type_commit(&usertype);
MPI_Type_free(&vtype);
MPI_Type_free(&rtype);
MPI_Type_get_extent(usertype, &lb, &extent);
userbufsz = extent;
userbuf = malloc(userbufsz);
cmpbuf = calloc(userbufsz, 1);
for (i=0; i< userbufsz/sizeof(int); i++) {
userbuf[i] = pi*i;
}
start = 10; acount = count*12;
ncmpi_begin_indep_data(ncid);
ncmpi_put_vara(ncid, varid, &start, &acount,
userbuf, 1, usertype);
ncmpi_close(ncid);
NC_CHECK(ncmpi_open(MPI_COMM_WORLD, "vectors.nc", NC_NOWRITE,
MPI_INFO_NULL, &ncid));
ncmpi_begin_indep_data(ncid);
NC_CHECK(ncmpi_inq_varid(ncid, "vector", &varid));
NC_CHECK(ncmpi_get_vara(ncid, varid, &start, &acount,
cmpbuf, 1, usertype));
ncmpi_close(ncid);
for (i=0; errs < 10 && i < acount; i++) {
/* vector of 4,3,5, so skip 4th and 5th items of every block */
if (i%STRIDE >= BLOCKLEN) continue;
if (userbuf[i] != cmpbuf[i]) {
errs++;
fprintf(stderr, "%d: expected 0x%x got 0x%x\n",
i, userbuf[i], cmpbuf[i]);
}
}
free(userbuf);
free(cmpbuf);
MPI_Type_free(&usertype);
MPI_Finalize();
return 0;
}
/**
* Read a single-precision parallel-nedcdf file.
*
* We assume here that localData is a scalar.
*
* Pnetcdf uses row-major format (same as FFTW).
*
* \param[in] filename : PnetCDF filename
* \param[in] starts : offset to where to start reading data
* \param[in] counts : number of elements read (3D sub-domain inside global)
* \param[in] gsizes : global sizes
* \param[out] localData : actual data buffer (size : nx*ny*nz*sizeof(float))
*
* localData must have been allocated prior to calling this routine.
*/
void read_pnetcdf(const std::string &filename,
MPI_Offset starts[3],
MPI_Offset counts[3],
int gsizes[3],
float *localData)
{
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
// netcdf file id
int ncFileId;
int err;
// file opening mode
int ncOpenMode = NC_NOWRITE;
int nbVar=1;
int varIds[nbVar];
MPI_Info mpi_info_used;
/*
* Open NetCDF file
*/
err = ncmpi_open(MPI_COMM_WORLD, filename.c_str(),
ncOpenMode,
MPI_INFO_NULL, &ncFileId);
if (err != NC_NOERR) {
printf("Error: ncmpi_open() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err));
MPI_Abort(MPI_COMM_WORLD, -1);
exit(1);
}
/*
* Query NetCDF mode
*/
int NC_mode;
err = ncmpi_inq_version(ncFileId, &NC_mode);
if (myRank==0) {
if (NC_mode == NC_64BIT_DATA)
std::cout << "Pnetcdf Input mode : NC_64BIT_DATA (CDF-5)\n";
else if (NC_mode == NC_64BIT_OFFSET)
std::cout << "Pnetcdf Input mode : NC_64BIT_OFFSET (CDF-2)\n";
else
std::cout << "Pnetcdf Input mode : unknown\n";
}
/*
* Query information about variable named "data"
*/
{
int ndims, nvars, ngatts, unlimited;
err = ncmpi_inq(ncFileId, &ndims, &nvars, &ngatts, &unlimited);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "data", &varIds[0]);
PNETCDF_HANDLE_ERROR;
}
/*
* Define expected data types (no conversion done here)
*/
MPI_Datatype mpiDataType = MPI_FLOAT;
/*
* Get all the MPI_IO hints used (just in case, we want to print it after
* reading data...
*/
err = ncmpi_get_file_info(ncFileId, &mpi_info_used);
PNETCDF_HANDLE_ERROR;
/*
* Read heavy data (take care of row-major / column major format !)
*/
int nItems = counts[IX]*counts[IY]*counts[IZ];
{
err = ncmpi_get_vara_all(ncFileId,
varIds[0],
starts,
counts,
localData,
nItems,
mpiDataType);
PNETCDF_HANDLE_ERROR;
} // end reading heavy data
/*
* close the file
*/
err = ncmpi_close(ncFileId);
PNETCDF_HANDLE_ERROR;
} // read_pnetcdf
int main(int argc, char **argv) {
int i, j;
int status;
int ncid1, ncid2;
int ndims, nvars, ngatts, unlimdimid;
char name[NC_MAX_NAME];
nc_type type, vartypes[NC_MAX_VARS];
MPI_Offset attlen;
MPI_Offset dimlen, shape[NC_MAX_VAR_DIMS], varsize, start[NC_MAX_VAR_DIMS];
void *valuep;
int dimids[NC_MAX_DIMS], varids[NC_MAX_VARS];
int vardims[NC_MAX_VARS][NC_MAX_VAR_DIMS/16]; /* divided by 16 due to my memory limitation */
int varndims[NC_MAX_VARS], varnatts[NC_MAX_VARS];
params opts;
int rank;
int nprocs;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
fprintf(stderr, "Testing independent read ... ");
parse_read_args(argc, argv, rank, &opts);
/********** START OF NETCDF ACCESS **************/
/* Read a netCDF file and write it out to another file */
/**
* Open the input dataset - ncid1:
* File name: "../data/test_int.nc"
* Dataset API: Collective
* And create the output dataset - ncid2:
* File name: "testread.nc"
* Dataset API: Collective
*/
status = ncmpi_open(comm, opts.infname, 0, MPI_INFO_NULL, &ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Inquire the dataset definitions of input dataset AND
* Add dataset definitions for output dataset.
*/
status = ncmpi_inq(ncid1, &ndims, &nvars, &ngatts, &unlimdimid);
if (status != NC_NOERR) handle_error(status);
/* Inquire global attributes, assume CHAR attributes. */
for (i = 0; i < ngatts; i++) {
status = ncmpi_inq_attname(ncid1, NC_GLOBAL, i, name);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_inq_att (ncid1, NC_GLOBAL, name, &type, &attlen);
if (status != NC_NOERR) handle_error(status);
switch (type) {
case NC_CHAR:
valuep = (void *)malloc(attlen * sizeof(char));
status = ncmpi_get_att_text(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_text (ncid2, NC_GLOBAL, name, attlen, (char *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_SHORT:
valuep = (void *)malloc(attlen * sizeof(short));
status = ncmpi_get_att_short(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_short (ncid2, NC_GLOBAL, name, type, attlen, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(attlen * sizeof(int));
status = ncmpi_get_att_int(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_int (ncid2, NC_GLOBAL, name, type, attlen, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(attlen * sizeof(float));
status = ncmpi_get_att_float(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_float (ncid2, NC_GLOBAL, name, type, attlen, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(attlen * sizeof(double));
status = ncmpi_get_att_double(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_double (ncid2, NC_GLOBAL, name, type, attlen, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
/* Inquire dimension */
for (i = 0; i < ndims; i++) {
status = ncmpi_inq_dim(ncid1, i, name, &dimlen);
if (status != NC_NOERR) handle_error(status);
if (i == unlimdimid)
dimlen = NC_UNLIMITED;
status = ncmpi_def_dim(ncid2, name, dimlen, dimids+i);
if (status != NC_NOERR) handle_error(status);
}
/* Inquire variables */
for (i = 0; i < nvars; i++) {
status = ncmpi_inq_var (ncid1, i, name, vartypes+i, varndims+i, vardims[i], varnatts+i);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var(ncid2, name, vartypes[i], varndims[i], vardims[i], varids+i);
if (status != NC_NOERR) handle_error(status);
/* var attributes, assume CHAR attributes */
for (j = 0; j < varnatts[i]; j++) {
status = ncmpi_inq_attname(ncid1, varids[i], j, name);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_inq_att (ncid1, varids[i], name, &type, &attlen);
if (status != NC_NOERR) handle_error(status);
switch (type) {
case NC_CHAR:
valuep = (void *)malloc(attlen * sizeof(char));
status = ncmpi_get_att_text(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_text (ncid2, varids[i], name, attlen, (char *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_SHORT:
valuep = (void *)malloc(attlen * sizeof(short));
status = ncmpi_get_att_short(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_short (ncid2, varids[i], name, type, attlen, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(attlen * sizeof(int));
status = ncmpi_get_att_int(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_int (ncid2, varids[i], name, type, attlen, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(attlen * sizeof(float));
status = ncmpi_get_att_float(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_float (ncid2, varids[i], name, type, attlen, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(attlen * sizeof(double));
status = ncmpi_get_att_double(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_double (ncid2, varids[i], name, type, attlen, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
}
/**
* End Define Mode (switch to data mode) for output dataset
* Dataset API: Collective
*/
status = ncmpi_enddef(ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Read data of variables from input dataset (assume INT variables)
* Write the data out to the corresponding variables in the output dataset
*
* Data Partition (Assume 4 processors):
* square: 2-D, (Block, *), 25*100 from 100*100
* cube: 3-D, (Block, *, *), 25*100*100 from 100*100*100
* xytime: 3-D, (Block, *, *), 25*100*100 from 100*100*100
* time: 1-D, Block-wise, 25 from 100
*
* Data Mode API: non-collective
*/
status = ncmpi_begin_indep_data(ncid1);
if (status != NC_NOERR) handle_error(status);
status =ncmpi_begin_indep_data(ncid2);
if (status != NC_NOERR) handle_error(status);
for (i = 0; i < NC_MAX_VAR_DIMS; i++)
start[i] = 0;
for (i = 0; i < nvars; i++) {
varsize = 1;
for (j = 0; j < varndims[i]; j++) {
status = ncmpi_inq_dim(ncid1, vardims[i][j], name, shape + j);
if (status != NC_NOERR) handle_error(status);
if (j == 0) {
shape[j] /= nprocs;
start[j] = shape[j] * rank;
}
varsize *= shape[j];
}
switch (vartypes[i]) {
case NC_CHAR:
break;
case NC_SHORT:
valuep = (void *)malloc(varsize * sizeof(short));
status = ncmpi_get_vara_short(ncid1, i, start, shape, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_short(ncid2, varids[i],
start, shape, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(varsize * sizeof(int));
status = ncmpi_get_vara_int(ncid1, i, start, shape, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int(ncid2, varids[i],
start, shape, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(varsize * sizeof(float));
status = ncmpi_get_vara_float(ncid1, i, start, shape, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_float(ncid2, varids[i],
start, shape, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(varsize * sizeof(double));
status = ncmpi_get_vara_double(ncid1, i, start, shape, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double(ncid2, varids[i],
start, shape, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
status = ncmpi_end_indep_data(ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_end_indep_data(ncid2);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_sync(ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_sync(ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Close the datasets
* Dataset API: collective
*/
status = ncmpi_close(ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_close(ncid2);
if (status != NC_NOERR) handle_error(status);
/******************* END OF NETCDF ACCESS ****************/
if (rank == 0)
fprintf(stderr, "OK\nInput file %s copied to: %s!\n", opts.infname, opts.outfname);
MPI_Finalize();
return 0;
}
int main(int argc, char **argv) {
int i, j=0, rank, nprocs, err;
int ncfile, ndims, nvars, ngatts, unlimited, var_ndims, var_natts;;
int *dimids=NULL;
char filename[256], varname[NC_MAX_NAME+1];
MPI_Offset *dim_sizes=NULL, var_size;
nc_type type;
int *data=NULL;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (rank == 0) printf("Usage: %s filename\n", argv[0]);
MPI_Finalize();
exit(-1);
}
if (argc > 1) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
if (rank == 0) {
err = ncmpi_open(MPI_COMM_SELF, filename,
NC_NOWRITE, MPI_INFO_NULL, &ncfile);
if (err != NC_NOERR) handle_error(err, __LINE__);
/* reader knows nothing about dataset, but we can interrogate with
* query routines: ncmpi_inq tells us how many of each kind of
* "thing" (dimension, variable, attribute) we will find in the
* file */
err = ncmpi_inq(ncfile, &ndims, &nvars, &ngatts, &unlimited);
if (err != NC_NOERR) handle_error(err, __LINE__);
/* we do not really need the name of the dimension or the variable
* for reading in this example. we could, in a different example,
* take the name of a variable on the command line and read just
* that one */
dim_sizes = (MPI_Offset*) calloc(ndims, sizeof(MPI_Offset));
/* netcdf dimension identifiers are allocated sequentially starting
* at zero; same for variable identifiers */
for(i=0; i<ndims; i++) {
err = ncmpi_inq_dimlen(ncfile, i, &(dim_sizes[i]) );
if (err != NC_NOERR) handle_error(err, __LINE__);
}
}
/* need to inform other MPI processors how many variables we will send */
err = MPI_Bcast(&nvars, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_ERR(err)
for(i=0; i<nvars; i++) {
/* rank 0 will find out the size of each variable, read it, and
* broadcast it to the rest of the processors */
if (rank == 0) {
/* obtain the number of dimensions of variable i, so we can
* allocate the dimids array */
err = ncmpi_inq_varndims(ncfile, i, &var_ndims);
if (err != NC_NOERR) handle_error(err, __LINE__);
dimids = (int*) malloc(var_ndims * sizeof(int));
err = ncmpi_inq_var(ncfile, i, varname, &type, &var_ndims, dimids,
&var_natts);
if (err != NC_NOERR) handle_error(err, __LINE__);
for (j=0, var_size=1; j<var_ndims; j++) {
var_size *= dim_sizes[dimids[j]];
}
free(dimids);
}
/* oddity: there's no predefined MPI_Offset type */
err = MPI_Bcast(&var_size, 1, MPI_OFFSET, 0, MPI_COMM_WORLD);
MPI_ERR(err)
data = (int*) calloc(var_size, sizeof(int));
if (rank == 0) {
switch(type) {
case NC_INT:
/* now we have the variable identifiers and we know how big
* they are */
/* this approach is not scalable: i/o happens from a single
* processor. This approach can be ok if the amount of
* data is quite small, but almost always the underlying
* MPI-IO library can do a better job */
err = ncmpi_get_var_int_all(ncfile, j, data);
if (err != NC_NOERR) handle_error(err, __LINE__);
break;
default:
/* we can do this for all the known netcdf types but this
* example is already getting too long */
fprintf(stderr, "unsupported NetCDF type \n");
}
}
/*and finally all processors have the data */
err = MPI_Bcast(data, var_size, MPI_INT, 0, MPI_COMM_WORLD);
MPI_ERR(err)
/* Here, every process can do computation on the local buffer, data,
or copy the contents to somewhere else */
free(data);
}
if (rank == 0) {
err = ncmpi_close(ncfile);
if (err != NC_NOERR) handle_error(err, __LINE__);
free(dim_sizes);
}
MPI_Finalize();
return 0;
}
