void read_pnetcdf(const std::string &filename,
int iVar,
ConfigMap &configMap,
HostArray<double> &localData)
{
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
int nbMpiProc;
MPI_Comm_size(MPI_COMM_WORLD, &nbMpiProc);
// netcdf file id
int ncFileId;
int err;
// ghostWidth
int ghostWidth = configMap.getInteger("mesh","ghostWidth",3);
// file creation mode
int ncOpenMode = NC_NOWRITE;
int varIds[8];
MPI_Offset starts[3], counts[3]; // read_size, sum_read_size;
MPI_Info mpi_info_used;
// domain local size
int nx,ny,nz;
// sizes to read
//int nx_r, ny_r, nz_r; // logical sizes / per sub-domain
//int nx_g, ny_g, nz_g; // sizes with ghost zones included / per sub-domain
/* read domain sizes */
nx=configMap.getInteger("mesh","nx",32);
ny=configMap.getInteger("mesh","ny",32);
nz=configMap.getInteger("mesh","nz",32);
// nx_g = nx+2*ghostWidth;
// ny_g = ny+2*ghostWidth;
// nz_g = nz+2*ghostWidth;
// get input filename from configMap
//std::string filename = configMap.getString("input", "filename", "");
/*
* Open NetCDF file
*/
err = ncmpi_open(MPI_COMM_WORLD, filename.c_str(),
ncOpenMode,
MPI_INFO_NULL, &ncFileId);
if (err != NC_NOERR) {
printf("Error: ncmpi_open() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err));
MPI_Abort(MPI_COMM_WORLD, -1);
exit(1);
}
/*
* Query NetCDF mode
*/
int NC_mode;
err = ncmpi_inq_version(ncFileId, &NC_mode);
if (myRank==0) {
if (NC_mode == NC_64BIT_DATA)
std::cout << "Pnetcdf Input mode : NC_64BIT_DATA (CDF-5)\n";
else if (NC_mode == NC_64BIT_OFFSET)
std::cout << "Pnetcdf Input mode : NC_64BIT_OFFSET (CDF-2)\n";
else
std::cout << "Pnetcdf Input mode : unknown\n";
}
/*
* Query information about variables
*/
{
int ndims, nvars, ngatts, unlimited;
err = ncmpi_inq(ncFileId, &ndims, &nvars, &ngatts, &unlimited);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho", &varIds[ID]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "E", &varIds[IP]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vx", &varIds[IU]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vy", &varIds[IV]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "rho_vz", &varIds[IW]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "Bx", &varIds[IA]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "By", &varIds[IB]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_inq_varid(ncFileId, "Bz", &varIds[IC]);
PNETCDF_HANDLE_ERROR;
} // end query information
/*
* Define expected data types (no conversion done here)
*/
//nc_type ncDataType;
MPI_Datatype mpiDataType;
//ncDataType = NC_DOUBLE;
mpiDataType = MPI_DOUBLE;
/*
* Get all the MPI_IO hints used (just in case, we want to print it after
* reading data...
*/
err = ncmpi_get_file_info(ncFileId, &mpi_info_used);
PNETCDF_HANDLE_ERROR;
/*
* Read heavy data (take care of row-major / column major format !)
*/
// use overlapping domains
// counts[IZ] = nx_rg;
// counts[IY] = ny_rg;
// counts[IX] = nz_rg;
// starts[IZ] = 0;
// starts[IY] = 0;
// starts[IX] = myRank*nz_r;
counts[IZ] = nx;
counts[IY] = ny;
counts[IX] = nz;
starts[IZ] = ghostWidth;
starts[IY] = ghostWidth;
starts[IX] = ghostWidth+myRank*nz;
int nItems = counts[IX]*counts[IY]*counts[IZ];
/*
* Actual reading
*/
{
double* data;
//data = &(localData(0,0,0,0));
data = localData.data();
err = ncmpi_get_vara_all(ncFileId, varIds[iVar],
starts, counts, data, nItems, mpiDataType);
PNETCDF_HANDLE_ERROR;
} // end for loop reading heavy data
/*
* close the file
*/
err = ncmpi_close(ncFileId);
PNETCDF_HANDLE_ERROR;
} // read_pnetcdf
/*
* adapted from HydroRunBaseMpi::outputPnetcdf
*
* assumes here that localData have size nx,ny,nz (no ghostWidth)
*
* see : test_pnetcdf_write.cpp
*
* Note that if ghostIncluded is false local_data must be sized upon nx,ny,nz
* if not size must be nx+2*ghostWidth,ny+2*ghostWidth,nz+2*ghostWidth
*
*/
void write_pnetcdf(const std::string &filename,
HostArray<double> &localData,
ConfigMap &configMap)
{
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
// read local domain sizes
int nx=configMap.getInteger("mesh","nx",32);
int ny=configMap.getInteger("mesh","ny",32);
int nz=configMap.getInteger("mesh","nz",32);
// read mpi geometry
int mx=configMap.getInteger("mpi","mx",1);
int my=configMap.getInteger("mpi","my",1);
int mz=configMap.getInteger("mpi","mz",1);
// MPI cartesian coordinates
// myRank = mpiCoord[0] + mx*mpiCoord[1] + mx*my*mpiCoord[2]
int mpiCoord[3];
{
mpiCoord[2] = myRank/(mx*my);
mpiCoord[1] = (myRank - mx*my*mpiCoord[2])/mx;
mpiCoord[0] = myRank - mx*my*mpiCoord[2] -mx*mpiCoord[1];
}
bool ghostIncluded = configMap.getBool("output", "ghostIncluded",false);
int ghostWidth = configMap.getInteger("mesh","ghostWidth",3);
// global size
int NX=nx*mx, NY=ny*my, NZ=nz*mz;
int gsizes[3];
gsizes[IZ] = NX;
gsizes[IY] = NY;
gsizes[IX] = NZ;
if ( ghostIncluded ) {
gsizes[IZ] += 2*ghostWidth;
gsizes[IY] += 2*ghostWidth;
gsizes[IX] += 2*ghostWidth;
}
// netcdf file id
int ncFileId;
int err;
// file creation mode
int ncCreationMode = NC_CLOBBER;
bool useCDF5 = configMap.getBool("output","pnetcdf_cdf5",false);
if (useCDF5)
ncCreationMode = NC_CLOBBER|NC_64BIT_DATA;
else // use CDF-2 file format
ncCreationMode = NC_CLOBBER|NC_64BIT_OFFSET;
// verbose log ?
bool pnetcdf_verbose = configMap.getBool("output","pnetcdf_verbose",false);
int nbVar=8;
int dimIds[3], varIds[nbVar];
MPI_Offset write_size, sum_write_size;
MPI_Info mpi_info_used;
char str[512];
// time measurement variables
double write_timing, max_write_timing, write_bw;
/*
* writing parameter (offset and size)
*/
MPI_Offset starts[3] = {0};
MPI_Offset counts[3] = {nz, ny, nx};
// take care that row-major / column major format
starts[IZ] = mpiCoord[IX]*nx;
starts[IY] = mpiCoord[IY]*ny;
starts[IX] = mpiCoord[IZ]*nz;
if ( ghostIncluded ) {
if ( mpiCoord[IX] == 0 )
counts[IZ] += ghostWidth;
if ( mpiCoord[IY] == 0 )
counts[IY] += ghostWidth;
if ( mpiCoord[IZ] == 0 )
counts[IX] += ghostWidth;
if ( mpiCoord[IX] == mx-1 )
counts[IZ] += ghostWidth;
if ( mpiCoord[IY] == my-1 )
counts[IY] += ghostWidth;
if ( mpiCoord[IZ] == mz-1 )
counts[IX] += ghostWidth;
starts[IZ] += ghostWidth;
starts[IY] += ghostWidth;
starts[IX] += ghostWidth;
if ( mpiCoord[IX] == 0 )
starts[IZ] -= ghostWidth;
if ( mpiCoord[IY] == 0 )
starts[IY] -= ghostWidth;
if ( mpiCoord[IZ] == 0 )
starts[IX] -= ghostWidth;
}
/*
* Create NetCDF file
*/
err = ncmpi_create(MPI_COMM_WORLD, filename.c_str(),
ncCreationMode,
MPI_INFO_NULL, &ncFileId);
if (err != NC_NOERR) {
printf("Error: ncmpi_create() file %s (%s)\n",filename.c_str(),ncmpi_strerror(err));
MPI_Abort(MPI_COMM_WORLD, -1);
exit(1);
}
/*
* Define dimensions
*/
err = ncmpi_def_dim(ncFileId, "x", gsizes[0], &dimIds[0]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_dim(ncFileId, "y", gsizes[1], &dimIds[1]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_dim(ncFileId, "z", gsizes[2], &dimIds[2]);
PNETCDF_HANDLE_ERROR;
/*
* Define variables
*/
nc_type ncDataType = NC_DOUBLE;
MPI_Datatype mpiDataType = MPI_DOUBLE;
err = ncmpi_def_var(ncFileId, "rho", ncDataType, 3, dimIds, &varIds[ID]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_var(ncFileId, "E", ncDataType, 3, dimIds, &varIds[IP]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_var(ncFileId, "rho_vx", ncDataType, 3, dimIds, &varIds[IU]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_var(ncFileId, "rho_vy", ncDataType, 3, dimIds, &varIds[IV]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_var(ncFileId, "rho_vz", ncDataType, 3, dimIds, &varIds[IW]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_var(ncFileId, "Bx", ncDataType, 3, dimIds, &varIds[IA]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_var(ncFileId, "By", ncDataType, 3, dimIds, &varIds[IB]);
PNETCDF_HANDLE_ERROR;
err = ncmpi_def_var(ncFileId, "Bz", ncDataType, 3, dimIds, &varIds[IC]);
PNETCDF_HANDLE_ERROR;
/*
* global attributes
*/
// did we use CDF-2 or CDF-5
{
int useCDF5_int = useCDF5 ? 1 : 0;
err = ncmpi_put_att_int(ncFileId, NC_GLOBAL, "CDF-5 mode", NC_INT, 1, &useCDF5_int);
PNETCDF_HANDLE_ERROR;
}
/*
* exit the define mode
*/
err = ncmpi_enddef(ncFileId);
PNETCDF_HANDLE_ERROR;
/*
* Get all the MPI_IO hints used
*/
err = ncmpi_get_file_info(ncFileId, &mpi_info_used);
PNETCDF_HANDLE_ERROR;
int nItems = counts[IX]*counts[IY]*counts[IZ];
for (int iVar=0; iVar<nbVar; iVar++) {
double *data = &(localData(0,0,0,iVar));
err = ncmpi_put_vara_all(ncFileId, varIds[iVar], starts, counts, data, nItems, mpiDataType);
PNETCDF_HANDLE_ERROR;
}
/*
* close the file
*/
err = ncmpi_close(ncFileId);
PNETCDF_HANDLE_ERROR;
} // write_pnetcdf