void read_pnetcdf(const std::string &filename,
int iVar,
ConfigMap &configMap,
HostArray<double> &localData)
{
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
int nbMpiProc;
MPI_Comm_size(MPI_COMM_WORLD, &nbMpiProc);
// netcdf file id
int ncFileId;
int err;
// ghostWidth
int ghostWidth = configMap.getInteger("mesh","ghostWidth",3);
// file creation mode
int ncOpenMode = NC_NOWRITE;
int varIds[8];
MPI_Offset starts[3], counts[3]; // read_size, sum_read_size;
MPI_Info mpi_info_used;
// domain local size
int nx,ny,nz;
// sizes to read
//int nx_r, ny_r, nz_r; // logical sizes / per sub-domain
//int nx_g, ny_g, nz_g; // sizes with ghost zones included / per sub-domain
/* read domain sizes */
nx=configMap.getInteger("mesh","nx",32);
ny=configMap.getInteger("mesh","ny",32);
nz=configMap.getInteger("mesh","nz",32);
// nx_g = nx+2*ghostWidth;
// ny_g = ny+2*ghostWidth;
// nz_g = nz+2*ghostWidth;
// get input filename from configMap
//std::string filename = configMap.getString("input", "filename", "");
/*
* Open NetCDF file
*/
err = ncmpi_open(MPI_COMM_WORLD, filename.c_str(),
ncOpenMode,
MPI_INFO_NULL, &ncFileId);
if (err != NC_NOERR) {
printf("Error: ncmpi_open() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err));
MPI_Abort(MPI_COMM_WORLD, -1);
exit(1);
}
/*
* Query NetCDF mode
*/
int NC_mode;
err = ncmpi_inq_version(ncFileId, &NC_mode);
if (myRank==0) {
if (NC_mode == NC_64BIT_DATA)
std::cout << "Pnetcdf Input mode : NC_64BIT_DATA (CDF-5)\n";
else if (NC_mode == NC_64BIT_OFFSET)
std::cout << "Pnetcdf Input mode : NC_64BIT_OFFSET (CDF-2)\n";
else
std::cout << "Pnetcdf Input mode : unknown\n";
}
/*
* Query information about variables
*/
{
int ndims, nvars, ngatts, unlimited;
err = ncmpi_inq(ncFileId, &ndims, &nvars, &ngatts, &unlimited);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho", &varIds[ID]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "E", &varIds[IP]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vx", &varIds[IU]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vy", &varIds[IV]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vz", &varIds[IW]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "Bx", &varIds[IA]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "By", &varIds[IB]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "Bz", &varIds[IC]);
PNETCDF_HANDLE_ERROR;
} // end query information
/*
* Define expected data types (no conversion done here)
*/
//nc_type ncDataType;
MPI_Datatype mpiDataType;
//ncDataType = NC_DOUBLE;
mpiDataType = MPI_DOUBLE;
/*
* Get all the MPI_IO hints used (just in case, we want to print it after
* reading data...
*/
err = ncmpi_get_file_info(ncFileId, &mpi_info_used);
PNETCDF_HANDLE_ERROR;
/*
* Read heavy data (take care of row-major / column major format !)
*/
// use overlapping domains
// counts[IZ] = nx_rg;
// counts[IY] = ny_rg;
// counts[IX] = nz_rg;
// starts[IZ] = 0;
// starts[IY] = 0;
// starts[IX] = myRank*nz_r;
counts[IZ] = nx;
counts[IY] = ny;
counts[IX] = nz;
starts[IZ] = ghostWidth;
starts[IY] = ghostWidth;
starts[IX] = ghostWidth+myRank*nz;
int nItems = counts[IX]*counts[IY]*counts[IZ];
/*
* Actual reading
*/
{
double* data;
//data = &(localData(0,0,0,0));
data = localData.data();
err = ncmpi_get_vara_all(ncFileId, varIds[iVar],
starts, counts, data, nItems, mpiDataType);
PNETCDF_HANDLE_ERROR;
} // end for loop reading heavy data
/*
* close the file
*/
err = ncmpi_close(ncFileId);
PNETCDF_HANDLE_ERROR;
} // read_pnetcdf
/**
* Read a single-precision parallel-nedcdf file.
*
* We assume here that localData is a scalar.
*
* Pnetcdf uses row-major format (same as FFTW).
*
* \param[in] filename : PnetCDF filename
* \param[in] starts : offset to where to start reading data
* \param[in] counts : number of elements read (3D sub-domain inside global)
* \param[in] gsizes : global sizes
* \param[out] localData : actual data buffer (size : nx*ny*nz*sizeof(float))
*
* localData must have been allocated prior to calling this routine.
*/
void read_pnetcdf(const std::string &filename,
MPI_Offset starts[3],
MPI_Offset counts[3],
int gsizes[3],
float *localData)
{
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
// netcdf file id
int ncFileId;
int err;
// file opening mode
int ncOpenMode = NC_NOWRITE;
int nbVar=1;
int varIds[nbVar];
MPI_Info mpi_info_used;
/*
* Open NetCDF file
*/
err = ncmpi_open(MPI_COMM_WORLD, filename.c_str(),
ncOpenMode,
MPI_INFO_NULL, &ncFileId);
if (err != NC_NOERR) {
printf("Error: ncmpi_open() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err));
MPI_Abort(MPI_COMM_WORLD, -1);
exit(1);
}
/*
* Query NetCDF mode
*/
int NC_mode;
err = ncmpi_inq_version(ncFileId, &NC_mode);
if (myRank==0) {
if (NC_mode == NC_64BIT_DATA)
std::cout << "Pnetcdf Input mode : NC_64BIT_DATA (CDF-5)\n";
else if (NC_mode == NC_64BIT_OFFSET)
std::cout << "Pnetcdf Input mode : NC_64BIT_OFFSET (CDF-2)\n";
else
std::cout << "Pnetcdf Input mode : unknown\n";
}
/*
* Query information about variable named "data"
*/
{
int ndims, nvars, ngatts, unlimited;
err = ncmpi_inq(ncFileId, &ndims, &nvars, &ngatts, &unlimited);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "data", &varIds[0]);
PNETCDF_HANDLE_ERROR;
}
/*
* Define expected data types (no conversion done here)
*/
MPI_Datatype mpiDataType = MPI_FLOAT;
/*
* Get all the MPI_IO hints used (just in case, we want to print it after
* reading data...
*/
err = ncmpi_get_file_info(ncFileId, &mpi_info_used);
PNETCDF_HANDLE_ERROR;
/*
* Read heavy data (take care of row-major / column major format !)
*/
int nItems = counts[IX]*counts[IY]*counts[IZ];
{
err = ncmpi_get_vara_all(ncFileId,
varIds[0],
starts,
counts,
localData,
nItems,
mpiDataType);
PNETCDF_HANDLE_ERROR;
} // end reading heavy data
/*
* close the file
*/
err = ncmpi_close(ncFileId);
PNETCDF_HANDLE_ERROR;
} // read_pnetcdf
int main(int argc, char **argv) {
int i, j;
int status;
int ncid1, ncid2;
int ndims, nvars, ngatts, unlimdimid;
char name[NC_MAX_NAME];
nc_type type, *vartypes;
MPI_Offset attlen, dimlen, varsize;
MPI_Offset *shape, *start;
void *valuep;
int *dimids, *varids, **vardims, *varndims, *varnatts;
params opts;
int rank;
int nprocs;
int NC_mode;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
fprintf(stderr, "Testing read ... ");
parse_read_args(argc, argv, rank, &opts);
/********** START OF NETCDF ACCESS **************/
/* Read a netCDF file and write it out to another file */
/**
* Open the input dataset - ncid1:
* File name: "../data/test_int.nc"
* Dataset API: Collective
* And create the output dataset - ncid2:
* File name: "testread.nc"
* Dataset API: Collective
*/
status = ncmpi_open(comm, opts.infname, 0, MPI_INFO_NULL, &ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_inq_version(ncid1, &NC_mode);
status = ncmpi_create(comm, opts.outfname, NC_CLOBBER|NC_mode, MPI_INFO_NULL, &ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Inquire the dataset definitions of input dataset AND
* Add dataset definitions for output dataset.
*/
status = ncmpi_inq(ncid1, &ndims, &nvars, &ngatts, &unlimdimid);
if (status != NC_NOERR) handle_error(status);
/* Inquire global attributes, assume CHAR attributes. */
for (i = 0; i < ngatts; i++) {
status = ncmpi_inq_attname(ncid1, NC_GLOBAL, i, name);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_inq_att (ncid1, NC_GLOBAL, name, &type, &attlen);
if (status != NC_NOERR) handle_error(status);
switch (type) {
case NC_CHAR:
valuep = (void *)malloc(attlen * sizeof(char));
status = ncmpi_get_att_text(ncid1, NC_GLOBAL, name, (char*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_text (ncid2, NC_GLOBAL, name, attlen, (char *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_SHORT:
valuep = (void *)malloc(attlen * sizeof(short));
status = ncmpi_get_att_short(ncid1, NC_GLOBAL, name, (short*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_short (ncid2, NC_GLOBAL, name, type, attlen, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(attlen * sizeof(int));
status = ncmpi_get_att_int(ncid1, NC_GLOBAL, name, (int*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_int (ncid2, NC_GLOBAL, name, type, attlen, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(attlen * sizeof(float));
status = ncmpi_get_att_float(ncid1, NC_GLOBAL, name, (float*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_float (ncid2, NC_GLOBAL, name, type, attlen, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(attlen * sizeof(double));
status = ncmpi_get_att_double(ncid1, NC_GLOBAL, name, (double*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_double (ncid2, NC_GLOBAL, name, type, attlen, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
/* Inquire dimension */
dimids = (int*) malloc(ndims * sizeof(int));
for (i = 0; i < ndims; i++) {
status = ncmpi_inq_dim(ncid1, i, name, &dimlen);
if (status != NC_NOERR) handle_error(status);
if (i == unlimdimid)
dimlen = NC_UNLIMITED;
status = ncmpi_def_dim(ncid2, name, dimlen, dimids+i);
if (status != NC_NOERR) handle_error(status);
}
free(dimids);
vartypes = (nc_type*) malloc(nvars * sizeof(nc_type));
varids = (int*) malloc(nvars * sizeof(int));
varndims = (int*) malloc(nvars * sizeof(int));
varnatts = (int*) malloc(nvars * sizeof(int));
vardims = (int**) malloc(nvars * sizeof(int*));
/* Inquire variables */
for (i = 0; i < nvars; i++) {
status = ncmpi_inq_varndims(ncid1, i, varndims+i);
if (status != NC_NOERR) handle_error(status);
vardims[i] = (int*) malloc(varndims[i] * sizeof(int));
status = ncmpi_inq_var (ncid1, i, name, vartypes+i, varndims+i, vardims[i], varnatts+i);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var(ncid2, name, vartypes[i], varndims[i], vardims[i], varids+i);
if (status != NC_NOERR) handle_error(status);
/* var attributes, assume CHAR attributes */
for (j = 0; j < varnatts[i]; j++) {
status = ncmpi_inq_attname(ncid1, i, j, name);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_inq_att (ncid1, i, name, &type, &attlen);
if (status != NC_NOERR) handle_error(status);
switch (type) {
case NC_CHAR:
valuep = (void *)malloc(attlen * sizeof(char));
status = ncmpi_get_att_text(ncid1, i, name, (char*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_text (ncid2, varids[i], name, attlen, (char *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_SHORT:
valuep = (void *)malloc(attlen * sizeof(short));
status = ncmpi_get_att_short(ncid1, i, name, (short*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_short (ncid2, varids[i], name, type, attlen, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(attlen * sizeof(int));
status = ncmpi_get_att_int(ncid1, i, name, (int*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_int (ncid2, varids[i], name, type, attlen, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(attlen * sizeof(float));
status = ncmpi_get_att_float(ncid1, i, name, (float*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_float (ncid2, varids[i], name, type, attlen, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(attlen * sizeof(double));
status = ncmpi_get_att_double(ncid1, i, name, (double*)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_double (ncid2, varids[i], name, type, attlen, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
}
/**
* End Define Mode (switch to data mode) for output dataset
* Dataset API: Collective
*/
status = ncmpi_enddef(ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Read data of variables from input dataset
* (ONLY DEAL WITH: NC_INT, NC_FLOAT, NC_DOUBLE for now)
* Write the data out to the corresponding variables in the output dataset
*
* Data Partition (Assume 4 processors):
* square: 2-D, (Block, *), 25*100 from 100*100
* cube: 3-D, (Block, *, *), 25*100*100 from 100*100*100
* xytime: 3-D, (Block, *, *), 25*100*100 from 100*100*100
* time: 1-D, Block-wise, 25 from 100
*
* Data Mode API: collective
*/
for (i = 0; i < nvars; i++) {
shape = (MPI_Offset*) calloc(varndims[i] * 2, sizeof(MPI_Offset));
start = shape + varndims[i];
varsize = 1;
for (j = 0; j < varndims[i]; j++) {
status = ncmpi_inq_dim(ncid1, vardims[i][j], name, shape + j);
if (status != NC_NOERR) handle_error(status);
if (j == 0) {
shape[j] /= nprocs;
start[j] = shape[j] * rank;
}
varsize *= shape[j];
}
switch (vartypes[i]) {
case NC_CHAR:
break;
case NC_SHORT:
valuep = (void *)malloc(varsize * sizeof(short));
status = ncmpi_get_vara_short_all(ncid1, i, start, shape, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_short_all(ncid2, varids[i],
start, shape, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(varsize * sizeof(int));
status = ncmpi_get_vara_int_all(ncid1, i, start, shape, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int_all(ncid2, varids[i],
start, shape, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(varsize * sizeof(float));
status = ncmpi_get_vara_float_all(ncid1, i, start, shape, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_float_all(ncid2, varids[i],
start, shape, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(varsize * sizeof(double));
status = ncmpi_get_vara_double_all(ncid1, i, start, shape, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double_all(ncid2, varids[i],
start, shape, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
free(vardims[i]);
free(shape);
}
/**
* Close the datasets
* Dataset API: collective
*/
status = ncmpi_close(ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_close(ncid2);
if (status != NC_NOERR) handle_error(status);
free(vartypes);
free(varids);
free(varndims);
free(varnatts);
free(vardims);
/******************* END OF NETCDF ACCESS ****************/
if (rank == 0)
fprintf(stderr, "OK\nInput file %s copied to: %s!\n", opts.infname, opts.outfname);
MPI_Finalize();
return 0;
}
