void ADIOI_Shfp_fname(ADIO_File fd, int rank, int *error_code)
{
int len;
char *slash, *ptr, tmp[PATH_MAX];
fd->shared_fp_fname = (char *) ADIOI_Malloc(PATH_MAX);
if (!rank) {
MPL_create_pathname(tmp, NULL, ".shfp", 0);
if (ADIOI_Strncpy(fd->shared_fp_fname, fd->filename, PATH_MAX)) {
*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname", fd->filename, ENAMETOOLONG);
return;
}
#ifdef ROMIO_NTFS
slash = strrchr(fd->filename, '\\');
#else
slash = strrchr(fd->filename, '/');
#endif
if (!slash) {
if (ADIOI_Strncpy(fd->shared_fp_fname, ".", 2)) {
*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname", fd->filename, ENAMETOOLONG);
return;
}
if (ADIOI_Strncpy(fd->shared_fp_fname + 1, fd->filename, PATH_MAX - 1)) {
*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname", fd->filename, ENAMETOOLONG);
return;
}
} else {
ptr = slash;
#ifdef ROMIO_NTFS
slash = strrchr(fd->shared_fp_fname, '\\');
#else
slash = strrchr(fd->shared_fp_fname, '/');
#endif
if (ADIOI_Strncpy(slash + 1, ".", 2)) {
*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname", fd->filename, ENAMETOOLONG);
return;
}
/* ok to cast: file names bounded by PATH_MAX and NAME_MAX */
len = (int) (PATH_MAX - (slash + 2 - fd->shared_fp_fname));
if (ADIOI_Strncpy(slash + 2, ptr + 1, len)) {
*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname", ptr + 1, ENAMETOOLONG);
return;
}
}
/* MPL_strnapp will return non-zero if truncated. That's ok */
MPL_strnapp(fd->shared_fp_fname, tmp, PATH_MAX);
len = (int) strlen(fd->shared_fp_fname);
MPI_Bcast(&len, 1, MPI_INT, 0, fd->comm);
MPI_Bcast(fd->shared_fp_fname, len + 1, MPI_CHAR, 0, fd->comm);
} else {
MPI_Bcast(&len, 1, MPI_INT, 0, fd->comm);
MPI_Bcast(fd->shared_fp_fname, len + 1, MPI_CHAR, 0, fd->comm);
}
}
void ADIOI_GEN_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
{
int err, rank;
static char myname[] = "ADIOI_GEN_RESIZE";
MPI_Comm_rank(fd->comm, &rank);
/* first aggregator performs ftruncate() */
if (rank == fd->hints->ranklist[0]) {
ADIOI_Assert(size == (off_t) size);
err = ftruncate(fd->fd_sys, (off_t)size);
}
/* bcast return value */
MPI_Bcast(&err, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (err == -1) {
*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
return;
}
/* --END ERROR HANDLING-- */
*error_code = MPI_SUCCESS;
}
void ADIOI_NFS_Open(ADIO_File fd, int *error_code)
{
int perm, amode;
mode_t old_mask;
static char myname[] = "ADIOI_NFS_OPEN";
if (fd->perm == ADIO_PERM_NULL) {
old_mask = umask(022);
umask(old_mask);
perm = old_mask ^ 0666;
}
else perm = fd->perm;
amode = 0;
if (fd->access_mode & ADIO_CREATE)
amode = amode | O_CREAT;
if (fd->access_mode & ADIO_RDONLY)
amode = amode | O_RDONLY;
if (fd->access_mode & ADIO_WRONLY)
amode = amode | O_WRONLY;
if (fd->access_mode & ADIO_RDWR)
amode = amode | O_RDWR;
if (fd->access_mode & ADIO_EXCL)
amode = amode | O_EXCL;
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_open_a, 0, NULL );
#endif
fd->fd_sys = open(fd->filename, amode, perm);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_open_b, 0, NULL );
#endif
fd->fd_direct = -1;
if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) {
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
#endif
fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
#endif
}
if (fd->fd_sys == -1) {
*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
}
else *error_code = MPI_SUCCESS;
}
/*@
ADIO_ResolveFileType - determines file system type and operations from
file name string; this is a collective call
Input Parameters:
. comm - communicator across which collective open is performed
. filename - name of file (string)
Output Parameters:
. fstype - (pointer to) int holding file system type
. ops - (address of) pointer to table of valid file operations
. error_code - (pointer to) int holding error code
Notes:
This code used to be in MPI_File_open(), but it has been moved into here in
order to clean things up. The goal is to separate all this "did we compile
for this fs type" code from the MPI layer and also to introduce the ADIOI_Fns
tables in a reasonable way. -- Rob, 06/06/2001
@*/
void ADIO_ResolveFileType(MPI_Comm comm, const char *filename, int *fstype,
ADIOI_Fns **ops, int *error_code)
{
int myerrcode, file_system, min_code, max_code;
char *tmp;
static char myname[] = "ADIO_RESOLVEFILETYPE";
char * p;
file_system = -1;
if (filename == NULL) {
*error_code = ADIOI_Err_create_code(myname, filename, ENOENT);
return;
}
tmp = strchr(filename, ':');
if (!tmp) {
int have_nfs_enabled=0;
*error_code = MPI_SUCCESS;
/* no prefix; use system-dependent function call to determine type */
/* Optimization: we can reduce the 'storm of stats' that result from
* thousands of mpi processes determinig file type this way. Let us
* have just one process stat the file and broadcast the result to
* everyone else.
* - Note that we will not catch cases like
* http://www.mcs.anl.gov/web-mail-archive/lists/mpich-discuss/2007/08/msg00042.html
* (edit: now http://lists.mcs.anl.gov/pipermail/mpich-discuss/2007-August/002648.html)
*
* where file systems are not mounted or available on other processes,
* but we'll catch those a few functions later in ADIO_Open
* - Note that if we have NFS enabled, we might have a situation where,
* for example, /home/user/data.out is UFS on one process but NFS on
* others, so we won't perform this optimization if NFS is enabled.
* - Another point: error codes and file system types are broadcast to
* all members of the communicator, so we get to skip the allreduce
* steps*/
#ifdef ROMIO_NFS
have_nfs_enabled=1;
#endif
if (!have_nfs_enabled) {
ADIO_FileSysType_fncall_scalable(comm, filename, &file_system, &myerrcode);
if (myerrcode != MPI_SUCCESS) {
*error_code = myerrcode;
return;
}
} else {
ADIO_FileSysType_fncall(filename, &file_system, &myerrcode);
/* the check for file system type will hang if any process got
* an error in ADIO_FileSysType_fncall. Processes encountering
* an error will return early, before the collective file
* system type check below. This case could happen if a full
* path exists on one node but not on others, and no prefix
* like ufs: was provided. see discussion at
* http://www.mcs.anl.gov/web-mail-archive/lists/mpich-discuss/2007/08/msg00042.html
* (edit: now
* http://lists.mcs.anl.gov/pipermail/mpich-discuss/2007-August/002648.html)
*/
MPI_Allreduce(&myerrcode, &max_code, 1, MPI_INT, MPI_MAX, comm);
if (max_code != MPI_SUCCESS) {
*error_code = max_code;
return;
}
/* ensure everyone came up with the same file system type */
MPI_Allreduce(&file_system, &min_code, 1, MPI_INT,
MPI_MIN, comm);
if (min_code == ADIO_NFS) file_system = ADIO_NFS;
}
}
else {
/* prefix specified; just match via prefix and assume everyone got
* the same thing.
*
* perhaps we should have this code go through the allreduce as well?
*/
ADIO_FileSysType_prefix(filename, &file_system, &myerrcode);
if (myerrcode != MPI_SUCCESS) {
*error_code = myerrcode;
return;
}
}
/* lastly, there may be situations where one cannot override the file
* system detection with a prefix -- maybe the file name is passed to both
* posix and MPI-IO routines, or maybe the file name is hard-coded into an
* application.
* Assumes all processes set the same environment varialble.
* Values: the same prefix you would stick on a file path. e.g. pvfs2: --
* including the colon! */
p = getenv("ROMIO_FSTYPE_FORCE");
if (p != NULL) {
ADIO_FileSysType_prefix(p, &file_system, &myerrcode);
if (myerrcode != MPI_SUCCESS) {
*error_code = myerrcode;
return;
}
}
/* verify that we support this file system type and set ops pointer */
if (file_system == ADIO_PFS) {
#ifndef ROMIO_PFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_PFS_operations;
#endif
}
if (file_system == ADIO_PIOFS) {
#ifndef ROMIO_PIOFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_PIOFS_operations;
#endif
}
if (file_system == ADIO_UFS) {
#ifndef ROMIO_UFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_UFS_operations;
#endif
}
if (file_system == ADIO_NFS) {
#ifndef ROMIO_NFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_NFS_operations;
#endif
}
if (file_system == ADIO_PANFS) {
#ifndef ROMIO_PANFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_PANFS_operations;
#endif
}
if (file_system == ADIO_HFS) {
#ifndef ROMIO_HFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_HFS_operations;
#endif
}
if (file_system == ADIO_XFS) {
#ifndef ROMIO_XFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_XFS_operations;
#endif
}
if (file_system == ADIO_SFS) {
#ifndef ROMIO_SFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_SFS_operations;
#endif
}
if (file_system == ADIO_PVFS) {
#ifndef ROMIO_PVFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_PVFS_operations;
#endif
}
if (file_system == ADIO_PVFS2) {
#ifndef ROMIO_PVFS2
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_PVFS2_operations;
#endif
}
if (file_system == ADIO_NTFS) {
#ifndef ROMIO_NTFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_NTFS_operations;
#endif
}
if (file_system == ADIO_TESTFS) {
#ifndef ROMIO_TESTFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_TESTFS_operations;
#endif
}
if (file_system == ADIO_GPFS) {
#ifndef ROMIO_GPFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_GPFS_operations;
#endif
}
if (file_system == ADIO_GRIDFTP) {
#ifndef ROMIO_GRIDFTP
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_GRIDFTP_operations;
#endif
}
if (file_system == ADIO_LUSTRE) {
#ifndef ROMIO_LUSTRE
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_LUSTRE_operations;
#endif
}
if (file_system == ADIO_ZOIDFS) {
#ifndef ROMIO_ZOIDFS
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**iofstypeunsupported", 0);
return;
#else
*ops = &ADIO_ZOIDFS_operations;
#endif
}
*error_code = MPI_SUCCESS;
*fstype = file_system;
return;
}
/*
ADIO_FileSysType_fncall - determines the file system type for a given file
using a system-dependent function call
Input Parameters:
. filename - pointer to file name character array
Output Parameters:
. fstype - location in which to store file system type (ADIO_XXX)
. error_code - location in which to store error code
MPI_SUCCESS is stored in the location pointed to by error_code on success.
This function is used by MPI_File_open() and MPI_File_delete() to determine
file system type. Most other functions use the type which is stored when the
file is opened.
*/
static void ADIO_FileSysType_fncall(const char *filename, int *fstype, int *error_code)
{
#if defined (ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE) || defined (HAVE_STRUCT_STATFS) || defined (ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE)
int err;
#endif
#ifdef ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
struct statvfs vfsbuf;
#endif
#ifdef HAVE_STRUCT_STATFS
struct statfs fsbuf;
#endif
#ifdef ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE
struct stat sbuf;
#endif
static char myname[] = "ADIO_RESOLVEFILETYPE_FNCALL";
/* NFS can get stuck and end up returing ESTALE "forever" */
#define MAX_ESTALE_RETRY 10000
int retry_cnt;
*error_code = MPI_SUCCESS;
#ifdef ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
/* rare: old solaris machines */
retry_cnt=0;
do {
err = statvfs(filename, &vfsbuf);
} while (err && (errno == ESTALE) && retry_cnt++ < MAX_ESTALE_RETRY);
if (err) {
/* ENOENT may be returned in two cases:
* 1) no directory entry for "filename"
* 2) "filename" is a dangling symbolic link
*
* ADIO_FileSysType_parentdir tries to deal with both cases.
*/
if (errno == ENOENT) {
char *dir;
ADIO_FileSysType_parentdir(filename, &dir);
err = statvfs(dir, &vfsbuf);
ADIOI_Free(dir);
}
else {
*error_code = ADIOI_Err_create_code(myname, filename, errno);
if(*error_code != MPI_SUCCESS) return;
}
}
/* --BEGIN ERROR HANDLING-- */
if (err) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
return;
}
/* --END ERROR HANDLING-- */
/* FPRINTF(stderr, "%s\n", vfsbuf.f_basetype); */
if (!strncmp(vfsbuf.f_basetype, "nfs", 3)) {
*fstype = ADIO_NFS;
return;
}
if (!strncmp(vfsbuf.f_basetype, "xfs", 3)) {
*fstype = ADIO_XFS;
return;
}
# ifdef ROMIO_UFS
/* if UFS support is enabled, default to that */
*fstype = ADIO_UFS;
return;
# endif
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
#endif /* STATVFS APPROACH */
#if defined(HAVE_STRUCT_STATFS) && defined(HAVE_STATFS)
/* common automagic fs-detection logic for any modern POSX-compliant
* environment */
retry_cnt = 0;
do {
err = statfs(filename, &fsbuf);
} while (err && (errno == ESTALE) && retry_cnt++ < MAX_ESTALE_RETRY);
if (err) {
if(errno == ENOENT) {
char *dir;
ADIO_FileSysType_parentdir(filename, &dir);
err = statfs(dir, &fsbuf);
ADIOI_Free(dir);
}
else {
*error_code = ADIOI_Err_create_code(myname, filename, errno);
if(*error_code != MPI_SUCCESS) return;
}
}
/* --BEGIN ERROR HANDLING-- */
if (err) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
return;
}
/* --END ERROR HANDLING-- */
# ifdef ROMIO_HAVE_STRUCT_STATFS_WITH_F_FSTYPENAME
/* uncommon: maybe only on Darwin ? */
if ( !strncmp("nfs",fsbuf.f_fstypename,3) ) {
*fstype = ADIO_NFS;
return;
}
# endif
#ifdef ROMIO_GPFS
if (fsbuf.f_type == GPFS_SUPER_MAGIC) {
*fstype = ADIO_GPFS;
return;
}
#endif
/* FPRINTF(stderr, "%d\n", fsbuf.f_type);*/
# ifdef NFS_SUPER_MAGIC
if (fsbuf.f_type == NFS_SUPER_MAGIC) {
*fstype = ADIO_NFS;
return;
}
# endif
#ifdef ROMIO_LUSTRE
# ifndef LL_SUPER_MAGIC
# define LL_SUPER_MAGIC 0x0BD00BD0
# endif
if (fsbuf.f_type == LL_SUPER_MAGIC) {
*fstype = ADIO_LUSTRE;
return;
}
#endif
# ifdef PAN_KERNEL_FS_CLIENT_SUPER_MAGIC
if (fsbuf.f_type == PAN_KERNEL_FS_CLIENT_SUPER_MAGIC) {
*fstype = ADIO_PANFS;
return;
}
# endif
# ifdef MOUNT_NFS
if (fsbuf.f_type == MOUNT_NFS) {
*fstype = ADIO_NFS;
return;
}
# endif
# ifdef MOUNT_PFS
if (fsbuf.f_type == MOUNT_PFS) {
*fstype = ADIO_PFS;
return;
}
# endif
# ifdef PVFS_SUPER_MAGIC
if (fsbuf.f_type == PVFS_SUPER_MAGIC) {
*fstype = ADIO_PVFS;
return;
}
# endif
# ifdef PVFS2_SUPER_MAGIC
if (fsbuf.f_type == PVFS2_SUPER_MAGIC) {
*fstype = ADIO_PVFS2;
return;
}
# endif
# ifdef XFS_SUPER_MAGIC
if (fsbuf.f_type == XFS_SUPER_MAGIC) {
*fstype = ADIO_XFS;
return;
}
# endif
# ifdef ROMIO_UFS
/* if UFS support is enabled, default to that */
*fstype = ADIO_UFS;
return;
# endif
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
#endif /* STATFS APPROACH */
#ifdef ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE
/* rare: maybe old NEC SX or SGI IRIX machines */
retry_cnt = 0;
do {
err = stat(filename, &sbuf);
} while (err && (errno == ESTALE) && retry_cnt++ < MAX_ESTALE_RETRY);
if (err) {
if(errno == ENOENT) {
char *dir;
ADIO_FileSysType_parentdir(filename, &dir);
err = stat(dir, &sbuf);
ADIOI_Free(dir);
}
else{
*error_code = ADIOI_Err_create_code(myname, filename, errno);
if(*error_code != MPI_SUCCESS) return;
}
}
if (err) {
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
return;
}
else {
if (!strcmp(sbuf.st_fstype, "nfs")) *fstype = ADIO_NFS;
else *fstype = ADIO_SFS; /* assuming SX4 for now */
}
#endif /* STAT APPROACH */
#ifdef ROMIO_NTFS
ADIOI_UNREFERENCED_ARG(filename);
ADIOI_UNREFERENCED_ARG(error_code);
*fstype = ADIO_NTFS; /* only supported FS on Windows */
#elif defined(ROMIO_NFS)
*fstype = ADIO_NFS;
#elif defined(ROMIO_UFS)
*fstype = ADIO_UFS;
#else
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
#endif
}
void ADIOI_GPFS_Open(ADIO_File fd, int *error_code)
{
int perm, old_mask, amode, rank, rc;
static char myname[] = "ADIOI_GPFS_OPEN";
/* set internal variables for tuning environment variables */
ad_gpfs_get_env_vars();
if (fd->perm == ADIO_PERM_NULL) {
old_mask = umask(022);
umask(old_mask);
perm = old_mask ^ 0666;
}
else perm = fd->perm;
amode = 0;
if (fd->access_mode & ADIO_CREATE)
amode = amode | O_CREAT;
if (fd->access_mode & ADIO_RDONLY)
amode = amode | O_RDONLY;
if (fd->access_mode & ADIO_WRONLY)
amode = amode | O_WRONLY;
if (fd->access_mode & ADIO_RDWR)
amode = amode | O_RDWR;
if (fd->access_mode & ADIO_EXCL)
amode = amode | O_EXCL;
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_open_a, 0, NULL);
#endif
fd->fd_sys = open(fd->filename, amode, perm);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_open_b, 0, NULL);
#endif
DBG_FPRINTF(stderr,"open('%s',%#X,%#X) rc=%d, errno=%d\n",fd->filename,amode,perm,fd->fd_sys,errno);
fd->fd_direct = -1;
if (gpfsmpio_devnullio == 1) {
fd->null_fd = open("/dev/null", O_RDWR);
} else {
fd->null_fd = -1;
}
if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND))
fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
if(fd->fd_sys != -1)
{
fd->blksize = 1048576; /* default to 1M */
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_stat_a, 0, NULL);
#endif
/* in this fs-specific routine, we might not be called over entire
* communicator (deferred open). Collect statistics on one process.
* ADIOI_GEN_Opencoll (common-code caller) will take care of the
* broadcast */
MPI_Comm_rank(fd->comm, &rank);
if ((rank == fd->hints->ranklist[0]) || (fd->comm == MPI_COMM_SELF)) {
struct stat64 gpfs_statbuf;
/* Get the (real) underlying file system block size */
rc = stat64(fd->filename, &gpfs_statbuf);
if (rc >= 0)
{
fd->blksize = gpfs_statbuf.st_blksize;
DBGV_FPRINTF(stderr,"Successful stat '%s'. Blocksize=%ld\n",
fd->filename,gpfs_statbuf.st_blksize);
}
else
{
DBGV_FPRINTF(stderr,"Stat '%s' failed with rc=%d, errno=%d\n",
fd->filename,rc,errno);
}
}
/* all other ranks have incorrect fd->blocksize, but ADIOI_GEN_Opencoll
* will take care of that in both standard and deferred-open case */
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_stat_b, 0, NULL);
#endif
#ifdef HAVE_GPFS_FCNTL_H
/* in parallel workload, might be helpful to immediately release block
* tokens. Or, system call overhead will outweigh any benefits... */
if (getenv("ROMIO_GPFS_FREE_LOCKS")!=NULL)
gpfs_free_all_locks(fd->fd_sys);
#endif
}
if (fd->fd_sys == -1) {
*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
}
else *error_code = MPI_SUCCESS;
}
void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
{
int perm, old_mask, amode, amode_direct;
int lumlen, myrank, flag, set_layout=0, err;
struct lov_user_md *lum = NULL;
char *value;
ADIO_Offset str_factor = -1, str_unit=0, start_iodev=-1;
size_t value_sz = (MPI_MAX_INFO_VAL+1)*sizeof(char);
#if defined(MPICH) || !defined(PRINT_ERR_MSG)
static char myname[] = "ADIOI_LUSTRE_OPEN";
#endif
MPI_Comm_rank(fd->comm, &myrank);
if (fd->perm == ADIO_PERM_NULL) {
old_mask = umask(022);
umask(old_mask);
perm = old_mask ^ 0666;
}
else perm = fd->perm;
amode = 0;
if (fd->access_mode & ADIO_CREATE)
amode = amode | O_CREAT;
if (fd->access_mode & ADIO_RDONLY)
amode = amode | O_RDONLY;
if (fd->access_mode & ADIO_WRONLY)
amode = amode | O_WRONLY;
if (fd->access_mode & ADIO_RDWR)
amode = amode | O_RDWR;
if (fd->access_mode & ADIO_EXCL)
amode = amode | O_EXCL;
amode_direct = amode | O_DIRECT;
/* odd length here because lov_user_md contains some fixed data and
* then a list of 'lmm_objects' representing stripe */
lumlen = sizeof(struct lov_user_md) +
MAX_LOV_UUID_COUNT * sizeof(struct lov_user_ost_data);
lum = (struct lov_user_md *)ADIOI_Calloc(1,lumlen);
value = (char *) ADIOI_Malloc(value_sz);
/* we already validated in LUSTRE_SetInfo that these are going to be the same */
if (fd->info != MPI_INFO_NULL) {
/* striping information */
ADIOI_Info_get(fd->info, "striping_unit", MPI_MAX_INFO_VAL,
value, &flag);
if (flag)
str_unit=atoll(value);
ADIOI_Info_get(fd->info, "striping_factor", MPI_MAX_INFO_VAL,
value, &flag);
if (flag)
str_factor=atoll(value);
ADIOI_Info_get(fd->info, "romio_lustre_start_iodevice",
MPI_MAX_INFO_VAL, value, &flag);
if (flag)
start_iodev=atoll(value);
}
if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0))
set_layout = 1;
/* if hints were set, we need to delay creation of any lustre objects.
* However, if we open the file with O_LOV_DELAY_CREATE and don't call the
* follow-up ioctl, subsequent writes will fail */
if (myrank == 0 && set_layout)
amode = amode | O_LOV_DELAY_CREATE;
fd->fd_sys = open(fd->filename, amode, perm);
if (fd->fd_sys == -1) goto fn_exit;
/* we can only set these hints on new files */
/* It was strange and buggy to open the file in the hint path. Instead,
* we'll apply the file tunings at open time */
if ((amode & O_CREAT) && set_layout ) {
/* if user has specified striping info, first aggregator tries to set
* it */
if (myrank == fd->hints->ranklist[0] || fd->comm == MPI_COMM_SELF) {
lum->lmm_magic = LOV_USER_MAGIC;
lum->lmm_pattern = 0;
/* crude check for overflow of lustre internal datatypes.
* Silently cap to large value if user provides a value
* larger than lustre supports */
if (str_unit > UINT_MAX)
lum->lmm_stripe_size = UINT_MAX;
else
lum->lmm_stripe_size = str_unit;
if (str_factor > USHRT_MAX)
lum->lmm_stripe_count = USHRT_MAX;
else
lum->lmm_stripe_count = str_factor;
if (start_iodev > USHRT_MAX)
lum->lmm_stripe_offset = USHRT_MAX;
else
lum->lmm_stripe_offset = start_iodev;
err = ioctl(fd->fd_sys, LL_IOC_LOV_SETSTRIPE, lum);
if (err == -1 && errno != EEXIST) {
fprintf(stderr, "Failure to set stripe info %s \n", strerror(errno));
/* not a fatal error, but user might care to know */
}
} /* End of striping parameters validation */
}
/* Pascal Deveze reports that, even though we pass a
* "GETSTRIPE" (read) flag to the ioctl, if some of the values of this
* struct are uninitialzed, the call can give an error. zero it out in case
* there are other members that must be initialized and in case
* lov_user_md struct changes in future */
memset(lum, 0, lumlen);
lum->lmm_magic = LOV_USER_MAGIC;
err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *)lum);
if (!err) {
fd->hints->striping_unit = lum->lmm_stripe_size;
MPL_snprintf(value, value_sz, "%d", lum->lmm_stripe_size);
ADIOI_Info_set(fd->info, "striping_unit", value);
fd->hints->striping_factor = lum->lmm_stripe_count;
MPL_snprintf(value, value_sz, "%d", lum->lmm_stripe_count);
ADIOI_Info_set(fd->info, "striping_factor", value);
fd->hints->start_iodevice = lum->lmm_stripe_offset;
MPL_snprintf(value, value_sz, "%d", lum->lmm_stripe_offset);
ADIOI_Info_set(fd->info, "romio_lustre_start_iodevice", value);
}
if (fd->access_mode & ADIO_APPEND)
fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
fd->fd_direct = -1;
if (fd->direct_write || fd->direct_read) {
fd->fd_direct = open(fd->filename, amode_direct, perm);
if (fd->fd_direct != -1) {
fd->d_mem = fd->d_miniosz = (1<<12);
} else {
perror("cannot open file with O_Direct");
fd->direct_write = fd->direct_read = 0;
}
}
fn_exit:
ADIOI_Free(lum);
ADIOI_Free(value);
/* --BEGIN ERROR HANDLING-- */
if (fd->fd_sys == -1 || ((fd->fd_direct == -1) &&
(fd->direct_write || fd->direct_read))) {
*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
}
/* --END ERROR HANDLING-- */
else *error_code = MPI_SUCCESS;
}
/*
ADIO_FileSysType_fncall - determines the file system type for a given file
using a system-dependent function call
Input Parameters:
. filename - pointer to file name character array
Output Parameters:
. fstype - location in which to store file system type (ADIO_XXX)
. error_code - location in which to store error code
MPI_SUCCESS is stored in the location pointed to by error_code on success.
This function is used by MPI_File_open() and MPI_File_delete() to determine
file system type. Most other functions use the type which is stored when the
file is opened.
*/
static void ADIO_FileSysType_fncall(const char *filename, int *fstype, int *error_code)
{
#if defined (ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE) || defined (HAVE_STRUCT_STATFS) || defined (ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE)
int err;
#endif
#ifdef ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
struct statvfs vfsbuf;
#endif
#ifdef HAVE_STRUCT_STATFS
struct statfs fsbuf;
#endif
#ifdef ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE
struct stat sbuf;
#endif
static char myname[] = "ADIO_RESOLVEFILETYPE_FNCALL";
*error_code = MPI_SUCCESS;
#ifdef ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
do {
err = statvfs(filename, &vfsbuf);
} while (err && (errno == ESTALE));
if (err) {
/* ENOENT may be returned in two cases:
* 1) no directory entry for "filename"
* 2) "filename" is a dangling symbolic link
*
* ADIO_FileSysType_parentdir tries to deal with both cases.
*/
if (errno == ENOENT) {
char *dir;
ADIO_FileSysType_parentdir(filename, &dir);
err = statvfs(dir, &vfsbuf);
ADIOI_Free(dir);
}
else {
*error_code = ADIOI_Err_create_code(myname, filename, errno);
if(*error_code != MPI_SUCCESS) return;
}
}
/* --BEGIN ERROR HANDLING-- */
if (err) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
return;
}
/* --END ERROR HANDLING-- */
/* FPRINTF(stderr, "%s\n", vfsbuf.f_basetype); */
if (!strncmp(vfsbuf.f_basetype, "nfs", 3)) {
*fstype = ADIO_NFS;
return;
}
if (!strncmp(vfsbuf.f_basetype, "xfs", 3)) {
*fstype = ADIO_XFS;
return;
}
# ifdef ROMIO_UFS
/* if UFS support is enabled, default to that */
*fstype = ADIO_UFS;
return;
# endif
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
#endif /* STATVFS APPROACH */
#ifdef HAVE_STRUCT_STATFS
do {
err = statfs(filename, &fsbuf);
} while (err && (errno == ESTALE));
if (err) {
if(errno == ENOENT) {
char *dir;
ADIO_FileSysType_parentdir(filename, &dir);
err = statfs(dir, &fsbuf);
ADIOI_Free(dir);
}
else {
*error_code = ADIOI_Err_create_code(myname, filename, errno);
if(*error_code != MPI_SUCCESS) return;
}
}
/* --BEGIN ERROR HANDLING-- */
if (err) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
return;
}
/* --END ERROR HANDLING-- */
# ifdef ROMIO_HAVE_STRUCT_STATFS_WITH_F_FSTYPENAME
if ( !strncmp("nfs",fsbuf.f_fstypename,3) ) {
*fstype = ADIO_NFS;
return;
}
# endif
# ifdef ROMIO_BGL
/* BlueGene is a special case: all file systems are AD_BGL, except for
* certain exceptions */
/* Bluegene needs to read enviroment variables before selecting the file system*/
ad_bgl_get_env_vars();
*fstype = ADIO_BGL;
check_for_lockless_exceptions(fsbuf.f_type, fstype);
*error_code = MPI_SUCCESS;
return;
# endif
/* FPRINTF(stderr, "%d\n", fsbuf.f_type);*/
# ifdef NFS_SUPER_MAGIC
if (fsbuf.f_type == NFS_SUPER_MAGIC) {
*fstype = ADIO_NFS;
return;
}
# endif
#ifdef ROMIO_LUSTRE
# ifndef LL_SUPER_MAGIC
# define LL_SUPER_MAGIC 0x0BD00BD0
# endif
if (fsbuf.f_type == LL_SUPER_MAGIC) {
*fstype = ADIO_LUSTRE;
return;
}
#endif
# ifdef PAN_KERNEL_FS_CLIENT_SUPER_MAGIC
if (fsbuf.f_type == PAN_KERNEL_FS_CLIENT_SUPER_MAGIC) {
*fstype = ADIO_PANFS;
return;
}
# endif
# ifdef MOUNT_NFS
if (fsbuf.f_type == MOUNT_NFS) {
*fstype = ADIO_NFS;
return;
}
# endif
# ifdef MOUNT_PFS
if (fsbuf.f_type == MOUNT_PFS) {
*fstype = ADIO_PFS;
return;
}
# endif
# ifdef PVFS_SUPER_MAGIC
if (fsbuf.f_type == PVFS_SUPER_MAGIC) {
*fstype = ADIO_PVFS;
return;
}
# endif
# ifdef PVFS2_SUPER_MAGIC
if (fsbuf.f_type == PVFS2_SUPER_MAGIC) {
*fstype = ADIO_PVFS2;
return;
}
# endif
# ifdef XFS_SUPER_MAGIC
if (fsbuf.f_type == XFS_SUPER_MAGIC) {
*fstype = ADIO_XFS;
return;
}
# endif
# ifdef ROMIO_UFS
/* if UFS support is enabled, default to that */
*fstype = ADIO_UFS;
return;
# endif
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
#endif /* STATFS APPROACH */
#ifdef ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE
do {
err = stat(filename, &sbuf);
} while (err && (errno == ESTALE));
if (err) {
if(errno == ENOENT) {
char *dir;
ADIO_FileSysType_parentdir(filename, &dir);
err = stat(dir, &sbuf);
ADIOI_Free(dir);
}
else{
*error_code = ADIOI_Err_create_code(myname, filename, errno);
if(*error_code != MPI_SUCCESS) return;
}
}
if (err) {
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
return;
}
else {
if (!strcmp(sbuf.st_fstype, "nfs")) *fstype = ADIO_NFS;
else *fstype = ADIO_SFS; /* assuming SX4 for now */
}
#endif /* STAT APPROACH */
#ifdef ROMIO_NTFS
ADIOI_UNREFERENCED_ARG(filename);
ADIOI_UNREFERENCED_ARG(error_code);
*fstype = ADIO_NTFS; /* only supported FS on Windows */
#elif defined(ROMIO_NFS)
*fstype = ADIO_NFS;
#elif defined(ROMIO_UFS)
*fstype = ADIO_UFS;
#else
/* --BEGIN ERROR HANDLING-- */
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
"**filename", "**filename %s", filename);
/* --END ERROR HANDLING-- */
#endif
}
