void *
IOR_Create_NCMPI(char * testFileName,
IOR_param_t * param)
{
int * fd;
int fd_mode;
MPI_Info mpiHints = MPI_INFO_NULL;
/* Wei-keng Liao: read and set MPI file hints from hintsFile */
SetHints(&mpiHints, param->hintsFileName);
if (rank == 0 && param->showHints) {
fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
fd = (int *)malloc(sizeof(int));
if (fd == NULL) ERR("Unable to malloc file descriptor");
fd_mode = GetFileMode(param);
NCMPI_CHECK(ncmpi_create(testComm, testFileName, fd_mode,
mpiHints, fd), "cannot create file");
/* Wei-keng Liao: print the MPI file hints currently used */
/* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release
if (rank == 0 && param->showHints) {
MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints),
"cannot get file info");
fprintf(stdout, "\nhints returned from opened file {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
*/
/* Wei-keng Liao: free up the mpiHints object */
/* WEL - this needs future fix from next release of PnetCDF
if (mpiHints != MPI_INFO_NULL)
MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
*/
return(fd);
} /* IOR_Create_NCMPI() */
/*
* Open a file through the HDF5 interface.
*/
static void *HDF5_Open(char *testFileName, IOR_param_t * param)
{
hid_t accessPropList, createPropList;
hsize_t memStart[NUM_DIMS],
dataSetDims[NUM_DIMS],
memStride[NUM_DIMS],
memCount[NUM_DIMS], memBlock[NUM_DIMS], memDataSpaceDims[NUM_DIMS];
int tasksPerDataSet;
unsigned fd_mode = (unsigned)0;
hid_t *fd;
MPI_Comm comm;
MPI_Info mpiHints = MPI_INFO_NULL;
fd = (hid_t *) malloc(sizeof(hid_t));
if (fd == NULL)
ERR("malloc() failed");
/*
* HDF5 uses different flags than those for POSIX/MPIIO
*/
if (param->open == WRITE) { /* WRITE flags */
param->openFlags = IOR_TRUNC;
} else { /* READ or check WRITE/READ flags */
param->openFlags = IOR_RDONLY;
}
/* set IOR file flags to HDF5 flags */
/* -- file open flags -- */
if (param->openFlags & IOR_RDONLY) {
fd_mode |= H5F_ACC_RDONLY;
}
if (param->openFlags & IOR_WRONLY) {
fprintf(stdout, "File write only not implemented in HDF5\n");
}
if (param->openFlags & IOR_RDWR) {
fd_mode |= H5F_ACC_RDWR;
}
if (param->openFlags & IOR_APPEND) {
fprintf(stdout, "File append not implemented in HDF5\n");
}
if (param->openFlags & IOR_CREAT) {
fd_mode |= H5F_ACC_CREAT;
}
if (param->openFlags & IOR_EXCL) {
fd_mode |= H5F_ACC_EXCL;
}
if (param->openFlags & IOR_TRUNC) {
fd_mode |= H5F_ACC_TRUNC;
}
if (param->openFlags & IOR_DIRECT) {
fprintf(stdout, "O_DIRECT not implemented in HDF5\n");
}
/* set up file creation property list */
createPropList = H5Pcreate(H5P_FILE_CREATE);
HDF5_CHECK(createPropList, "cannot create file creation property list");
/* set size of offset and length used to address HDF5 objects */
HDF5_CHECK(H5Pset_sizes
(createPropList, sizeof(hsize_t), sizeof(hsize_t)),
"cannot set property list properly");
/* set up file access property list */
accessPropList = H5Pcreate(H5P_FILE_ACCESS);
HDF5_CHECK(accessPropList, "cannot create file access property list");
/*
* someday HDF5 implementation will allow subsets of MPI_COMM_WORLD
*/
/* store MPI communicator info for the file access property list */
if (param->filePerProc) {
comm = MPI_COMM_SELF;
} else {
comm = testComm;
}
SetHints(&mpiHints, param->hintsFileName);
/*
* note that with MP_HINTS_FILTERED=no, all key/value pairs will
* be in the info object. The info object that is attached to
* the file during MPI_File_open() will only contain those pairs
* deemed valid by the implementation.
*/
/* show hints passed to file */
if (rank == 0 && param->showHints) {
fprintf(stdout, "\nhints passed to access property list {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
HDF5_CHECK(H5Pset_fapl_mpio(accessPropList, comm, mpiHints),
"cannot set file access property list");
/* set alignment */
HDF5_CHECK(H5Pset_alignment(accessPropList, param->setAlignment,
param->setAlignment),
"cannot set alignment");
/* open file */
if (param->open == WRITE) { /* WRITE */
*fd = H5Fcreate(testFileName, fd_mode,
createPropList, accessPropList);
HDF5_CHECK(*fd, "cannot create file");
} else { /* READ or CHECK */
*fd = H5Fopen(testFileName, fd_mode, accessPropList);
HDF5_CHECK(*fd, "cannot open file");
}
/* show hints actually attached to file handle */
if (param->showHints || (1) /* WEL - this needs fixing */ ) {
if (rank == 0
&& (param->showHints) /* WEL - this needs fixing */ ) {
WARN("showHints not working for HDF5");
}
} else {
MPI_Info mpiHintsCheck = MPI_INFO_NULL;
hid_t apl;
apl = H5Fget_access_plist(*fd);
HDF5_CHECK(H5Pget_fapl_mpio(apl, &comm, &mpiHintsCheck),
"cannot get info object through HDF5");
if (rank == 0) {
fprintf(stdout,
"\nhints returned from opened file (HDF5) {\n");
ShowHints(&mpiHintsCheck);
fprintf(stdout, "}\n");
if (1 == 1) { /* request the MPIIO file handle and its hints */
MPI_File *fd_mpiio;
HDF5_CHECK(H5Fget_vfd_handle
(*fd, apl, (void **)&fd_mpiio),
"cannot get MPIIO file handle");
MPI_CHECK(MPI_File_get_info
(*fd_mpiio, &mpiHintsCheck),
"cannot get info object through MPIIO");
fprintf(stdout,
"\nhints returned from opened file (MPIIO) {\n");
ShowHints(&mpiHintsCheck);
fprintf(stdout, "}\n");
}
}
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
}
/* this is necessary for resetting various parameters
needed for reopening and checking the file */
newlyOpenedFile = TRUE;
HDF5_CHECK(H5Pclose(createPropList),
"cannot close creation property list");
HDF5_CHECK(H5Pclose(accessPropList),
"cannot close access property list");
/* create property list for serial/parallel access */
xferPropList = H5Pcreate(H5P_DATASET_XFER);
HDF5_CHECK(xferPropList, "cannot create transfer property list");
/* set data transfer mode */
if (param->collective) {
HDF5_CHECK(H5Pset_dxpl_mpio(xferPropList, H5FD_MPIO_COLLECTIVE),
"cannot set collective data transfer mode");
} else {
HDF5_CHECK(H5Pset_dxpl_mpio
(xferPropList, H5FD_MPIO_INDEPENDENT),
"cannot set independent data transfer mode");
}
/* set up memory data space for transfer */
memStart[0] = (hsize_t) 0;
memCount[0] = (hsize_t) 1;
memStride[0] = (hsize_t) (param->transferSize / sizeof(IOR_size_t));
memBlock[0] = (hsize_t) (param->transferSize / sizeof(IOR_size_t));
memDataSpaceDims[0] = (hsize_t) param->transferSize;
memDataSpace = H5Screate_simple(NUM_DIMS, memDataSpaceDims, NULL);
HDF5_CHECK(memDataSpace, "cannot create simple memory data space");
/* define hyperslab for memory data space */
HDF5_CHECK(H5Sselect_hyperslab(memDataSpace, H5S_SELECT_SET,
memStart, memStride, memCount,
memBlock), "cannot create hyperslab");
/* set up parameters for fpp or different dataset count */
if (param->filePerProc) {
tasksPerDataSet = 1;
} else {
if (param->individualDataSets) {
/* each task in segment has single data set */
tasksPerDataSet = 1;
} else {
/* share single data set across all tasks in segment */
tasksPerDataSet = param->numTasks;
}
}
dataSetDims[0] = (hsize_t) ((param->blockSize / sizeof(IOR_size_t))
* tasksPerDataSet);
/* create a simple data space containing information on size
and shape of data set, and open it for access */
dataSpace = H5Screate_simple(NUM_DIMS, dataSetDims, NULL);
HDF5_CHECK(dataSpace, "cannot create simple data space");
return (fd);
}
/*
* Open a file through the MPIIO interface. Setup file view.
*/
static void *MPIIO_Open(char *testFileName, IOR_param_t * param)
{
int fd_mode = (int)0,
offsetFactor,
tasksPerFile,
transfersPerBlock = param->blockSize / param->transferSize;
struct fileTypeStruct {
int globalSizes[2], localSizes[2], startIndices[2];
} fileTypeStruct;
MPI_File *fd;
MPI_Comm comm;
MPI_Info mpiHints = MPI_INFO_NULL;
fd = (MPI_File *) malloc(sizeof(MPI_File));
if (fd == NULL)
ERR("malloc failed()");
*fd = 0;
/* set IOR file flags to MPIIO flags */
/* -- file open flags -- */
if (param->openFlags & IOR_RDONLY) {
fd_mode |= MPI_MODE_RDONLY;
}
if (param->openFlags & IOR_WRONLY) {
fd_mode |= MPI_MODE_WRONLY;
}
if (param->openFlags & IOR_RDWR) {
fd_mode |= MPI_MODE_RDWR;
}
if (param->openFlags & IOR_APPEND) {
fd_mode |= MPI_MODE_APPEND;
}
if (param->openFlags & IOR_CREAT) {
fd_mode |= MPI_MODE_CREATE;
}
if (param->openFlags & IOR_EXCL) {
fd_mode |= MPI_MODE_EXCL;
}
if (param->openFlags & IOR_TRUNC) {
fprintf(stdout, "File truncation not implemented in MPIIO\n");
}
if (param->openFlags & IOR_DIRECT) {
fprintf(stdout, "O_DIRECT not implemented in MPIIO\n");
}
/*
* MPI_MODE_UNIQUE_OPEN mode optimization eliminates the overhead of file
* locking. Only open a file in this mode when the file will not be con-
* currently opened elsewhere, either inside or outside the MPI environment.
*/
fd_mode |= MPI_MODE_UNIQUE_OPEN;
if (param->filePerProc) {
comm = MPI_COMM_SELF;
} else {
comm = testComm;
}
SetHints(&mpiHints, param->hintsFileName);
/*
* note that with MP_HINTS_FILTERED=no, all key/value pairs will
* be in the info object. The info object that is attached to
* the file during MPI_File_open() will only contain those pairs
* deemed valid by the implementation.
*/
/* show hints passed to file */
if (rank == 0 && param->showHints) {
fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
MPI_CHECK(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd),
"cannot open file");
/* show hints actually attached to file handle */
if (rank == 0 && param->showHints) {
MPI_CHECK(MPI_File_get_info(*fd, &mpiHints),
"cannot get file info");
fprintf(stdout, "\nhints returned from opened file {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
/* preallocate space for file */
if (param->preallocate && param->open == WRITE) {
MPI_CHECK(MPI_File_preallocate(*fd,
(MPI_Offset) (param->segmentCount
*
param->blockSize *
param->numTasks)),
"cannot preallocate file");
}
/* create file view */
if (param->useFileView) {
/* create contiguous transfer datatype */
MPI_CHECK(MPI_Type_contiguous
(param->transferSize / sizeof(IOR_size_t),
MPI_LONG_LONG_INT, ¶m->transferType),
"cannot create contiguous datatype");
MPI_CHECK(MPI_Type_commit(¶m->transferType),
"cannot commit datatype");
if (param->filePerProc) {
offsetFactor = 0;
tasksPerFile = 1;
} else {
offsetFactor = (rank + rankOffset) % param->numTasks;
tasksPerFile = param->numTasks;
}
/*
* create file type using subarray
*/
fileTypeStruct.globalSizes[0] = 1;
fileTypeStruct.globalSizes[1] =
transfersPerBlock * tasksPerFile;
fileTypeStruct.localSizes[0] = 1;
fileTypeStruct.localSizes[1] = transfersPerBlock;
fileTypeStruct.startIndices[0] = 0;
fileTypeStruct.startIndices[1] =
transfersPerBlock * offsetFactor;
MPI_CHECK(MPI_Type_create_subarray
(2, fileTypeStruct.globalSizes,
fileTypeStruct.localSizes,
fileTypeStruct.startIndices, MPI_ORDER_C,
param->transferType, ¶m->fileType),
"cannot create subarray");
MPI_CHECK(MPI_Type_commit(¶m->fileType),
"cannot commit datatype");
MPI_CHECK(MPI_File_set_view(*fd, (MPI_Offset) 0,
param->transferType,
param->fileType, "native",
(MPI_Info) MPI_INFO_NULL),
"cannot set file view");
}
return ((void *)fd);
}
