void ADIOI_PFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
char *value, *value_in_fd;
int flag, tmp_val, str_factor=-1, str_unit=-1, start_iodev=-1;
struct sattr attr;
int err, myrank, fd_sys, perm, amode, old_mask;
if ( (fd->info) == MPI_INFO_NULL) {
/* This must be part of the open call. can set striping parameters
if necessary. */
MPI_Info_create(&(fd->info));
/* has user specified striping or server buffering parameters
and do they have the same value on all processes? */
if (users_info != MPI_INFO_NULL) {
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
str_factor=atoi(value);
tmp_val = str_factor;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != str_factor) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"striping_factor",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
str_unit=atoi(value);
tmp_val = str_unit;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != str_unit) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"striping_unit",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
ADIOI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
start_iodev=atoi(value);
tmp_val = start_iodev;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != start_iodev) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"start_iodevice",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
/* if user has specified striping info, process 0 tries to set it */
if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) {
MPI_Comm_rank(fd->comm, &myrank);
if (!myrank) {
if (fd->perm == ADIO_PERM_NULL) {
old_mask = umask(022);
umask(old_mask);
perm = old_mask ^ 0666;
}
else perm = fd->perm;
amode = 0;
if (fd->access_mode & ADIO_CREATE)
amode = amode | O_CREAT;
if (fd->access_mode & ADIO_RDONLY)
amode = amode | O_RDONLY;
if (fd->access_mode & ADIO_WRONLY)
amode = amode | O_WRONLY;
if (fd->access_mode & ADIO_RDWR)
amode = amode | O_RDWR;
if (fd->access_mode & ADIO_EXCL)
amode = amode | O_EXCL;
fd_sys = open(fd->filename, amode, perm);
err = fcntl(fd_sys, F_GETSATTR, &attr);
if (!err) {
if (str_unit > 0) attr.s_sunitsize = str_unit;
if ((start_iodev >= 0) &&
(start_iodev < attr.s_sfactor))
attr.s_start_sdir = start_iodev;
if ((str_factor > 0) && (str_factor < attr.s_sfactor))
attr.s_sfactor = str_factor;
err = fcntl(fd_sys, F_SETSATTR, &attr);
}
close(fd_sys);
}
MPI_Barrier(fd->comm);
}
/* Has user asked for pfs server buffering to be turned on?
If so, mark it as true in fd->info and turn it on in
ADIOI_PFS_Open after the file is opened */
ADIOI_Info_get(users_info, "pfs_svr_buf", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && (!strcmp(value, "true")))
ADIOI_Info_set(fd->info, "pfs_svr_buf", "true");
else ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");
ADIOI_Free(value);
}
else ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");
/* set the values for collective I/O and data sieving parameters */
ADIOI_GEN_SetInfo(fd, users_info, error_code);
}
else {
/* The file has been opened previously and fd->fd_sys is a valid
file descriptor. cannot set striping parameters now. */
/* set the values for collective I/O and data sieving parameters */
ADIOI_GEN_SetInfo(fd, users_info, error_code);
/* has user specified value for pfs_svr_buf? */
if (users_info != MPI_INFO_NULL) {
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
ADIOI_Info_get(users_info, "pfs_svr_buf", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && (!strcmp(value, "true") || !strcmp(value, "false"))) {
value_in_fd = (char *)
ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
ADIOI_Info_get(fd->info, "pfs_svr_buf", MPI_MAX_INFO_VAL,
value_in_fd, &flag);
if (strcmp(value, value_in_fd)) {
if (!strcmp(value, "true")) {
err = fcntl(fd->fd_sys, F_PFS_SVR_BUF, TRUE);
if (!err)
ADIOI_Info_set(fd->info, "pfs_svr_buf", "true");
}
else {
err = fcntl(fd->fd_sys, F_PFS_SVR_BUF, FALSE);
if (!err)
ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");
}
}
ADIOI_Free(value_in_fd);
}
ADIOI_Free(value);
}
}
*error_code = MPI_SUCCESS;
}
void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
char *value;
int flag;
ADIO_Offset stripe_val[3], str_factor = -1, str_unit = 0, start_iodev = -1;
int myrank;
static char myname[] = "ADIOI_LUSTRE_SETINFO";
#ifdef HAVE_LUSTRE_LOCKAHEAD
/* Set lock ahead default hints */
fd->hints->fs_hints.lustre.lock_ahead_read = 0;
fd->hints->fs_hints.lustre.lock_ahead_write = 0;
fd->hints->fs_hints.lustre.lock_ahead_num_extents = 500;
fd->hints->fs_hints.lustre.lock_ahead_flags = 0;
#endif
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
if ((fd->info) == MPI_INFO_NULL) {
/* This must be part of the open call. can set striping parameters
* if necessary. */
MPI_Info_create(&(fd->info));
ADIOI_Info_set(fd->info, "direct_read", "false");
ADIOI_Info_set(fd->info, "direct_write", "false");
fd->direct_read = fd->direct_write = 0;
/* initialize lustre hints */
ADIOI_Info_set(fd->info, "romio_lustre_co_ratio", "1");
fd->hints->fs_hints.lustre.co_ratio = 1;
ADIOI_Info_set(fd->info, "romio_lustre_coll_threshold", "0");
fd->hints->fs_hints.lustre.coll_threshold = 0;
ADIOI_Info_set(fd->info, "romio_lustre_ds_in_coll", "enable");
fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_ENABLE;
/* has user specified striping or server buffering parameters
* and do they have the same value on all processes? */
if (users_info != MPI_INFO_NULL) {
/* striping information */
ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
ADIOI_Info_set(fd->info, "striping_unit", value);
str_unit = atoll(value);
}
ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
ADIOI_Info_set(fd->info, "striping_factor", value);
str_factor = atoll(value);
}
ADIOI_Info_get(users_info, "romio_lustre_start_iodevice",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
ADIOI_Info_set(fd->info, "romio_lustre_start_iodevice", value);
start_iodev = atoll(value);
}
/* direct read and write */
ADIOI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL, value, &flag);
if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
ADIOI_Info_set(fd->info, "direct_read", "true");
fd->direct_read = 1;
}
ADIOI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL, value, &flag);
if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
ADIOI_Info_set(fd->info, "direct_write", "true");
fd->direct_write = 1;
}
#ifdef HAVE_LUSTRE_LOCKAHEAD
/* Get lock ahead hints */
ADIOI_Info_check_and_install_int(fd, users_info,
"romio_lustre_cb_lock_ahead_write",
&(fd->hints->fs_hints.lustre.lock_ahead_write),
myname, error_code);
ADIOI_Info_check_and_install_int(fd, users_info,
"romio_lustre_cb_lock_ahead_read",
&(fd->hints->fs_hints.lustre.lock_ahead_read),
myname, error_code);
/* If, and only if, we're using lock ahead,
* process/set the number of extents to pre-lock and the flags */
if (fd->hints->fs_hints.lustre.lock_ahead_read ||
fd->hints->fs_hints.lustre.lock_ahead_write) {
/* Get user's number of extents */
ADIOI_Info_check_and_install_int(fd, users_info,
"romio_lustre_cb_lock_ahead_num_extents",
&(fd->hints->fs_hints.
lustre.lock_ahead_num_extents), myname,
error_code);
/* ADIOI_Info_check_and_install_int doesn't set the
* value in fd unless it was in user_info, but knowing
* the value - default or explicit - is useful.
* Set the final number of extents in the fd->info */
MPL_snprintf(value, MPI_MAX_INFO_VAL + 1, "%d",
fd->hints->fs_hints.lustre.lock_ahead_num_extents);
ADIOI_Info_set(fd->info, "romio_lustre_cb_lock_ahead_num_extents", value);
/* Get user's flags */
ADIOI_Info_check_and_install_int(fd, users_info,
"romio_lustre_cb_lock_ahead_flags",
&(fd->hints->fs_hints.lustre.lock_ahead_flags),
myname, error_code);
}
#endif
}
/* set striping information with ioctl */
MPI_Comm_rank(fd->comm, &myrank);
if (myrank == 0) {
stripe_val[0] = str_factor;
stripe_val[1] = str_unit;
stripe_val[2] = start_iodev;
}
MPI_Bcast(stripe_val, 3, MPI_OFFSET, 0, fd->comm);
/* do not open file in hint processing. Open file in open routines,
* where we can better deal with EXCL flag . Continue to check the
* "all processors set a value" condition holds. */
if (stripe_val[0] != str_factor
|| stripe_val[1] != str_unit || stripe_val[2] != start_iodev) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME("ADIOI_LUSTRE_SetInfo",
"str_factor or str_unit or start_iodev", error_code);
ADIOI_Free(value);
return;
}
}
/* get other hint */
if (users_info != MPI_INFO_NULL) {
/* CO: IO Clients/OST,
* to keep the load balancing between clients and OSTs */
ADIOI_Info_check_and_install_int(fd, users_info, "romio_lustre_co_ratio",
&(fd->hints->fs_hints.lustre.co_ratio), myname,
error_code);
/* coll_threshold:
* if the req size is bigger than this, collective IO may not be performed.
*/
ADIOI_Info_check_and_install_int(fd, users_info, "romio_lustre_coll_threshold",
&(fd->hints->fs_hints.lustre.coll_threshold), myname,
error_code);
/* ds_in_coll: disable data sieving in collective IO */
ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_lustre_ds_in_coll",
&(fd->hints->fs_hints.lustre.ds_in_coll), myname,
error_code);
}
/* set the values for collective I/O and data sieving parameters */
ADIOI_GEN_SetInfo(fd, users_info, error_code);
/* generic hints might step on striping_unit */
if (users_info != MPI_INFO_NULL) {
ADIOI_Info_check_and_install_int(fd, users_info, "striping_unit", NULL, myname, error_code);
}
if (ADIOI_Direct_read)
fd->direct_read = 1;
if (ADIOI_Direct_write)
fd->direct_write = 1;
ADIOI_Free(value);
*error_code = MPI_SUCCESS;
}
void ADIOI_GEN_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
/* if fd->info is null, create a new info object.
Initialize fd->info to default values.
Initialize fd->hints to default values.
Examine the info object passed by the user. If it contains values that
ROMIO understands, override the default. */
MPI_Info info;
char *value;
int flag, intval, tmp_val, nprocs=0, nprocs_is_valid = 0, len;
static char myname[] = "ADIOI_GEN_SETINFO";
if (fd->info == MPI_INFO_NULL) MPI_Info_create(&(fd->info));
info = fd->info;
/* Note that fd->hints is allocated at file open time; thus it is
* not necessary to allocate it, or check for allocation, here.
*/
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
if (value == NULL) {
/* NEED TO HANDLE ENOMEM */
}
/* initialize info and hints to default values if they haven't been
* previously initialized
*/
if (!fd->hints->initialized) {
/* buffer size for collective I/O */
MPI_Info_set(info, "cb_buffer_size", ADIOI_CB_BUFFER_SIZE_DFLT);
fd->hints->cb_buffer_size = atoi(ADIOI_CB_BUFFER_SIZE_DFLT);
/* default is to let romio automatically decide when to use
* collective buffering
*/
MPI_Info_set(info, "romio_cb_read", "automatic");
fd->hints->cb_read = ADIOI_HINT_AUTO;
MPI_Info_set(info, "romio_cb_write", "automatic");
fd->hints->cb_write = ADIOI_HINT_AUTO;
fd->hints->cb_config_list = NULL;
/* number of processes that perform I/O in collective I/O */
MPI_Comm_size(fd->comm, &nprocs);
nprocs_is_valid = 1;
ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", nprocs);
MPI_Info_set(info, "cb_nodes", value);
fd->hints->cb_nodes = nprocs;
/* hint indicating that no indep. I/O will be performed on this file */
MPI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->no_indep_rw = 0;
/* deferred_open derrived from no_indep_rw and cb_{read,write} */
fd->hints->deferred_open = 0;
/* buffer size for data sieving in independent reads */
MPI_Info_set(info, "ind_rd_buffer_size", ADIOI_IND_RD_BUFFER_SIZE_DFLT);
fd->hints->ind_rd_buffer_size = atoi(ADIOI_IND_RD_BUFFER_SIZE_DFLT);
/* buffer size for data sieving in independent writes */
MPI_Info_set(info, "ind_wr_buffer_size", ADIOI_IND_WR_BUFFER_SIZE_DFLT);
fd->hints->ind_wr_buffer_size = atoi(ADIOI_IND_WR_BUFFER_SIZE_DFLT);
/* default is to let romio automatically decide when to use data
* sieving
*/
MPI_Info_set(info, "romio_ds_read", "automatic");
fd->hints->ds_read = ADIOI_HINT_AUTO;
MPI_Info_set(info, "romio_ds_write", "automatic");
fd->hints->ds_write = ADIOI_HINT_AUTO;
fd->hints->initialized = 1;
}
/* add in user's info if supplied */
if (users_info != MPI_INFO_NULL) {
MPI_Info_get(users_info, "cb_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval=atoi(value)) > 0)) {
tmp_val = intval;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != intval) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"cb_buffer_size",
error_code);
return;
}
/* --END ERROR HANDLING-- */
MPI_Info_set(info, "cb_buffer_size", value);
fd->hints->cb_buffer_size = intval;
}
/* new hints for enabling/disabling coll. buffering on
* reads/writes
*/
MPI_Info_get(users_info, "romio_cb_read", MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
MPI_Info_set(info, "romio_cb_read", value);
fd->hints->cb_read = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
/* romio_cb_read overrides no_indep_rw */
MPI_Info_set(info, "romio_cb_read", value);
MPI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->cb_read = ADIOI_HINT_DISABLE;
fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
MPI_Info_set(info, "romio_cb_read", value);
fd->hints->cb_read = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->cb_read;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->cb_read) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_read",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
MPI_Info_get(users_info, "romio_cb_write", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
MPI_Info_set(info, "romio_cb_write", value);
fd->hints->cb_write = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE"))
{
/* romio_cb_write overrides no_indep_rw, too */
MPI_Info_set(info, "romio_cb_write", value);
MPI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->cb_write = ADIOI_HINT_DISABLE;
fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") ||
!strcmp(value, "AUTOMATIC"))
{
MPI_Info_set(info, "romio_cb_write", value);
fd->hints->cb_write = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->cb_write;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->cb_write) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_write",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
/* new hint for specifying no indep. read/write will be performed */
MPI_Info_get(users_info, "romio_no_indep_rw", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "true") || !strcmp(value, "TRUE")) {
/* if 'no_indep_rw' set, also hint that we will do
* collective buffering: if we aren't doing independent io,
* then we have to do collective */
MPI_Info_set(info, "romio_no_indep_rw", value);
MPI_Info_set(info, "romio_cb_write", "enable");
MPI_Info_set(info, "romio_cb_read", "enable");
fd->hints->no_indep_rw = 1;
fd->hints->cb_read = 1;
fd->hints->cb_write = 1;
tmp_val = 1;
}
else if (!strcmp(value, "false") || !strcmp(value, "FALSE")) {
MPI_Info_set(info, "romio_no_indep_rw", value);
fd->hints->no_indep_rw = 0;
tmp_val = 0;
}
else {
/* default is above */
tmp_val = 0;
}
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->no_indep_rw) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_no_indep_rw",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
/* new hints for enabling/disabling data sieving on
* reads/writes
*/
MPI_Info_get(users_info, "romio_ds_read", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
MPI_Info_set(info, "romio_ds_read", value);
fd->hints->ds_read = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
MPI_Info_set(info, "romio_ds_read", value);
fd->hints->ds_read = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
MPI_Info_set(info, "romio_ds_read", value);
fd->hints->ds_read = ADIOI_HINT_AUTO;
}
/* otherwise ignore */
}
MPI_Info_get(users_info, "romio_ds_write", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
MPI_Info_set(info, "romio_ds_write", value);
fd->hints->ds_write = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
MPI_Info_set(info, "romio_ds_write", value);
fd->hints->ds_write = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
MPI_Info_set(info, "romio_ds_write", value);
fd->hints->ds_write = ADIOI_HINT_AUTO;
}
/* otherwise ignore */
}
MPI_Info_get(users_info, "cb_nodes", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval=atoi(value)) > 0)) {
tmp_val = intval;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != intval) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"cb_nodes",
error_code);
return;
}
/* --END ERROR HANDLING-- */
if (!nprocs_is_valid) {
/* if hints were already initialized, we might not
* have already gotten this?
*/
MPI_Comm_size(fd->comm, &nprocs);
nprocs_is_valid = 1;
}
if (intval <= nprocs) {
MPI_Info_set(info, "cb_nodes", value);
fd->hints->cb_nodes = intval;
}
}
MPI_Info_get(users_info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval = atoi(value)) > 0)) {
MPI_Info_set(info, "ind_wr_buffer_size", value);
fd->hints->ind_wr_buffer_size = intval;
}
MPI_Info_get(users_info, "ind_rd_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval = atoi(value)) > 0)) {
MPI_Info_set(info, "ind_rd_buffer_size", value);
fd->hints->ind_rd_buffer_size = intval;
}
MPI_Info_get(users_info, "cb_config_list", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
if (fd->hints->cb_config_list == NULL) {
/* only set cb_config_list if it isn't already set.
* Note that since we set it below, this ensures that
* the cb_config_list hint will be set at file open time
* either by the user or to the default
*/
MPI_Info_set(info, "cb_config_list", value);
len = (strlen(value)+1) * sizeof(char);
fd->hints->cb_config_list = ADIOI_Malloc(len);
if (fd->hints->cb_config_list == NULL) {
/* NEED TO HANDLE ENOMEM */
}
ADIOI_Strncpy(fd->hints->cb_config_list, value, len);
}
/* if it has been set already, we ignore it the second time.
* otherwise we would get an error if someone used the same
* info value with a cb_config_list value in it in a couple
* of calls, which would be irritating. */
}
}
/* handle cb_config_list default value here; avoids an extra
* free/alloc and insures it is always set
*/
if (fd->hints->cb_config_list == NULL) {
MPI_Info_set(info, "cb_config_list", ADIOI_CB_CONFIG_LIST_DFLT);
len = (strlen(ADIOI_CB_CONFIG_LIST_DFLT)+1) * sizeof(char);
fd->hints->cb_config_list = ADIOI_Malloc(len);
if (fd->hints->cb_config_list == NULL) {
/* NEED TO HANDLE ENOMEM */
}
ADIOI_Strncpy(fd->hints->cb_config_list, ADIOI_CB_CONFIG_LIST_DFLT, len);
}
/* deferred_open won't be set by callers, but if the user doesn't
* explicitly disable collecitve buffering (two-phase) and does hint that
* io w/o independent io is going on, we'll set this internal hint as a
* convenience */
if ( ( (fd->hints->cb_read != ADIOI_HINT_DISABLE) \
&& (fd->hints->cb_write != ADIOI_HINT_DISABLE)\
&& fd->hints->no_indep_rw ) ) {
fd->hints->deferred_open = 1;
} else {
/* setting romio_no_indep_rw enable and romio_cb_{read,write}
* disable at the same time doesn't make sense. honor
* romio_cb_{read,write} and force the no_indep_rw hint to
* 'disable' */
MPI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->no_indep_rw = 0;
fd->hints->deferred_open = 0;
}
if ((fd->file_system == ADIO_PIOFS) || (fd->file_system == ADIO_PVFS) ||
(fd->file_system == ADIO_PVFS2) ) {
/* no data sieving for writes in PIOFS, PVFS and PVFS2, because they do not
support file locking */
MPI_Info_get(info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
/* get rid of this value if it is set */
MPI_Info_delete(info, "ind_wr_buffer_size");
}
/* note: leave ind_wr_buffer_size alone; used for other cases
* as well. -- Rob Ross, 04/22/2003
*/
MPI_Info_set(info, "romio_ds_write", "disable");
fd->hints->ds_write = ADIOI_HINT_DISABLE;
}
ADIOI_Free(value);
*error_code = MPI_SUCCESS;
}
void ADIOI_PVFS2_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
char *value;
int flag, tmp_value;
static char myname[] = "ADIOI_PVFS_SETINFO";
if ((fd->info) == MPI_INFO_NULL) {
/* part of the open call */
MPI_Info_create(&(fd->info));
ADIOI_Info_set(fd->info, "romio_pvfs2_debugmask", "0");
fd->hints->fs_hints.pvfs2.debugmask = 0;
ADIOI_Info_set(fd->info, "striping_factor", "0");
fd->hints->striping_factor = 0;
ADIOI_Info_set(fd->info, "striping_unit", "0");
fd->hints->striping_unit = 0;
/* disable the aggressive strided optimizations by default */
ADIOI_Info_set(fd->info, "romio_pvfs2_posix_read", "disable");
ADIOI_Info_set(fd->info, "romio_pvfs2_posix_write", "disable");
fd->hints->fs_hints.pvfs2.posix_read = ADIOI_HINT_DISABLE;
fd->hints->fs_hints.pvfs2.posix_write = ADIOI_HINT_DISABLE;
ADIOI_Info_set(fd->info, "romio_pvfs2_dtype_read", "disable");
ADIOI_Info_set(fd->info, "romio_pvfs2_dtype_write", "disable");
fd->hints->fs_hints.pvfs2.dtype_read = ADIOI_HINT_DISABLE;
fd->hints->fs_hints.pvfs2.dtype_write = ADIOI_HINT_DISABLE;
ADIOI_Info_set(fd->info, "romio_pvfs2_listio_read", "disable");
ADIOI_Info_set(fd->info, "romio_pvfs2_listio_write", "disable");
fd->hints->fs_hints.pvfs2.listio_read = ADIOI_HINT_DISABLE;
fd->hints->fs_hints.pvfs2.listio_write = ADIOI_HINT_DISABLE;
/* any user-provided hints? */
if (users_info != MPI_INFO_NULL) {
/* pvfs2 debugging */
value = (char *) ADIOI_Malloc( (MPI_MAX_INFO_VAL+1)*sizeof(char));
ADIOI_Info_get(users_info, "romio_pvfs2_debugmask",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
tmp_value = fd->hints->fs_hints.pvfs2.debugmask =
PVFS_debug_eventlog_to_mask(value);
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_value != fd->hints->fs_hints.pvfs2.debugmask) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_pvfs2_debugmask",
error_code);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Info_set(fd->info, "romio_pvfs2_debugmask", value);
}
/* the striping factor */
ADIOI_Info_get(users_info, "striping_factor",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
tmp_value = fd->hints->striping_factor = atoi(value);
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_value != fd->hints->striping_factor) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"striping_factor",
error_code);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Info_set(fd->info, "striping_factor", value);
}
/* the striping unit */
ADIOI_Info_get(users_info, "striping_unit",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
tmp_value = fd->hints->striping_unit = atoi(value);
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_value != fd->hints->striping_unit) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"striping_unit",
error_code);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Info_set(fd->info, "striping_unit", value);
}
/* distribution name */
ADIOI_Info_get(users_info, "romio_pvfs2_distribution_name",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
}
/* POSIX read */
ADIOI_Info_get(users_info, "romio_pvfs2_posix_read",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs2_posix_read", value);
fd->hints->fs_hints.pvfs2.posix_read = ADIOI_HINT_ENABLE;
}
else if ( !strcmp(value, "disable") ||
!strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info , "romio_pvfs2_posix_read", value);
fd->hints->fs_hints.pvfs2.posix_read = ADIOI_HINT_DISABLE;
}
tmp_value = fd->hints->fs_hints.pvfs2.posix_read;
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
if (tmp_value != fd->hints->fs_hints.pvfs2.posix_read) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"posix_read",
error_code);
return;
}
}
/* POSIX write */
ADIOI_Info_get(users_info, "romio_pvfs2_posix_write",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs2_posix_write", value);
fd->hints->fs_hints.pvfs2.posix_write = ADIOI_HINT_ENABLE;
}
else if ( !strcmp(value, "disable") ||
!strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info , "romio_pvfs2_posix_write", value);
fd->hints->fs_hints.pvfs2.posix_write = ADIOI_HINT_DISABLE;
}
tmp_value = fd->hints->fs_hints.pvfs2.posix_write;
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
if (tmp_value != fd->hints->fs_hints.pvfs2.posix_write) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"posix_write",
error_code);
return;
}
}
/* Datatype read */
ADIOI_Info_get(users_info, "romio_pvfs2_dtype_read",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs2_dtype_read", value);
fd->hints->fs_hints.pvfs2.dtype_read = ADIOI_HINT_ENABLE;
}
else if ( !strcmp(value, "disable") ||
!strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info , "romio_pvfs2_dtype_read", value);
fd->hints->fs_hints.pvfs2.dtype_read = ADIOI_HINT_DISABLE;
}
tmp_value = fd->hints->fs_hints.pvfs2.dtype_read;
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
if (tmp_value != fd->hints->fs_hints.pvfs2.dtype_read) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"dtype_read",
error_code);
return;
}
}
/* Datatype write */
ADIOI_Info_get(users_info, "romio_pvfs2_dtype_write",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs2_dtype_write", value);
fd->hints->fs_hints.pvfs2.dtype_write = ADIOI_HINT_ENABLE;
}
else if ( !strcmp(value, "disable") ||
!strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info , "romio_pvfs2_dtype_write", value);
fd->hints->fs_hints.pvfs2.dtype_write = ADIOI_HINT_DISABLE;
}
tmp_value = fd->hints->fs_hints.pvfs2.dtype_write;
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
if (tmp_value != fd->hints->fs_hints.pvfs2.dtype_write) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"dtype_write",
error_code);
return;
}
}
/* Listio read */
ADIOI_Info_get(users_info, "romio_pvfs2_listio_read",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs2_listio_read", value);
fd->hints->fs_hints.pvfs2.listio_read = ADIOI_HINT_ENABLE;
}
else if ( !strcmp(value, "disable") ||
!strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info , "romio_pvfs2_listio_read", value);
fd->hints->fs_hints.pvfs2.listio_read = ADIOI_HINT_DISABLE;
}
tmp_value = fd->hints->fs_hints.pvfs2.listio_read;
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
if (tmp_value != fd->hints->fs_hints.pvfs2.listio_read) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"listio_read",
error_code);
return;
}
}
/* Datatype write */
ADIOI_Info_get(users_info, "romio_pvfs2_listio_write",
MPI_MAX_INFO_VAL, value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs2_listio_write", value);
fd->hints->fs_hints.pvfs2.listio_write = ADIOI_HINT_ENABLE;
}
else if ( !strcmp(value, "disable") ||
!strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info , "romio_pvfs2_listio_write", value);
fd->hints->fs_hints.pvfs2.listio_write = ADIOI_HINT_DISABLE;
}
tmp_value = fd->hints->fs_hints.pvfs2.listio_write;
MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
if (tmp_value != fd->hints->fs_hints.pvfs2.listio_write) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"listio_write",
error_code);
return;
}
}
ADIOI_Free(value);
}
}
/* set the values for collective I/O and data sieving parameters */
ADIOI_GEN_SetInfo(fd, users_info, error_code);
*error_code = MPI_SUCCESS;
}
void ADIOI_BGL_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
/* if fd->info is null, create a new info object.
Initialize fd->info to default values.
Initialize fd->hints to default values.
Examine the info object passed by the user. If it contains values that
ROMIO understands, override the default. */
MPI_Info info;
char *value;
int flag, intval, tmp_val, nprocs=0, nprocs_is_valid = 0;
static char myname[] = "ADIOI_BGL_SETINFO";
int did_anything = 0;
if (fd->info == MPI_INFO_NULL) MPI_Info_create(&(fd->info));
info = fd->info;
/* Note that fd->hints is allocated at file open time; thus it is
* not necessary to allocate it, or check for allocation, here.
*/
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
AD_BGL_assert ((value != NULL));
/* initialize info and hints to default values if they haven't been
* previously initialized
*/
if (!fd->hints->initialized) {
did_anything = 1;
/* buffer size for collective I/O */
ADIOI_Info_set(info, "cb_buffer_size", ADIOI_BGL_CB_BUFFER_SIZE_DFLT);
fd->hints->cb_buffer_size = atoi(ADIOI_BGL_CB_BUFFER_SIZE_DFLT);
/* default is to let romio automatically decide when to use
* collective buffering
*/
ADIOI_Info_set(info, "romio_cb_read", "enable");
fd->hints->cb_read = ADIOI_HINT_ENABLE;
ADIOI_Info_set(info, "romio_cb_write", "enable");
fd->hints->cb_write = ADIOI_HINT_ENABLE;
if ( fd->hints->cb_config_list != NULL ) ADIOI_Free (fd->hints->cb_config_list);
fd->hints->cb_config_list = NULL;
/* number of processes that perform I/O in collective I/O */
MPI_Comm_size(fd->comm, &nprocs);
nprocs_is_valid = 1;
ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", nprocs);
ADIOI_Info_set(info, "cb_nodes", value);
fd->hints->cb_nodes = -1;
/* hint indicating that no indep. I/O will be performed on this file */
ADIOI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->no_indep_rw = 0;
/* bgl is not implementing file realms (ADIOI_IOStridedColl),
initialize to disabled it. */
/* hint instructing the use of persistent file realms */
ADIOI_Info_set(info, "romio_cb_pfr", "disable");
fd->hints->cb_pfr = ADIOI_HINT_DISABLE;
/* hint guiding the assignment of persistent file realms */
ADIOI_Info_set(info, "romio_cb_fr_types", "aar");
fd->hints->cb_fr_type = ADIOI_FR_AAR;
/* hint to align file realms with a certain byte value */
ADIOI_Info_set(info, "romio_cb_fr_alignment", "1");
fd->hints->cb_fr_alignment = 1;
/* hint to set a threshold percentage for a datatype's size/extent at
* which data sieving should be done in collective I/O */
ADIOI_Info_set(info, "romio_cb_ds_threshold", "0");
fd->hints->cb_ds_threshold = 0;
/* hint to switch between point-to-point or all-to-all for two-phase */
ADIOI_Info_set(info, "romio_cb_alltoall", "automatic");
fd->hints->cb_alltoall = ADIOI_HINT_AUTO;
/* deferred_open derived from no_indep_rw and cb_{read,write} */
fd->hints->deferred_open = 0;
/* buffer size for data sieving in independent reads */
ADIOI_Info_set(info, "ind_rd_buffer_size", ADIOI_BGL_IND_RD_BUFFER_SIZE_DFLT);
fd->hints->ind_rd_buffer_size = atoi(ADIOI_BGL_IND_RD_BUFFER_SIZE_DFLT);
/* buffer size for data sieving in independent writes */
ADIOI_Info_set(info, "ind_wr_buffer_size", ADIOI_BGL_IND_WR_BUFFER_SIZE_DFLT);
fd->hints->ind_wr_buffer_size = atoi(ADIOI_BGL_IND_WR_BUFFER_SIZE_DFLT);
if(fd->file_system == ADIO_UFS)
{
/* default for ufs/pvfs is to disable data sieving */
ADIOI_Info_set(info, "romio_ds_read", "disable");
fd->hints->ds_read = ADIOI_HINT_DISABLE;
ADIOI_Info_set(info, "romio_ds_write", "disable");
fd->hints->ds_write = ADIOI_HINT_DISABLE;
}
else
{
/* default is to let romio automatically decide when to use data
* sieving
*/
ADIOI_Info_set(info, "romio_ds_read", "automatic");
fd->hints->ds_read = ADIOI_HINT_AUTO;
ADIOI_Info_set(info, "romio_ds_write", "automatic");
fd->hints->ds_write = ADIOI_HINT_AUTO;
}
/* still to do: tune this a bit for a variety of file systems. there's
* no good default value so just leave it unset */
fd->hints->min_fdomain_size = 0;
fd->hints->striping_unit = 0;
fd->hints->initialized = 1;
}
/* add in user's info if supplied */
if (users_info != MPI_INFO_NULL) {
ADIOI_Info_get(users_info, "cb_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval=atoi(value)) > 0)) {
tmp_val = intval;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != intval) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"cb_buffer_size",
error_code);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Info_set(info, "cb_buffer_size", value);
fd->hints->cb_buffer_size = intval;
}
#if 0
/* bgl is not implementing file realms (ADIOI_IOStridedColl) ... */
/* aligning file realms to certain sizes (e.g. stripe sizes)
* may benefit I/O performance */
ADIOI_Info_get(users_info, "romio_cb_fr_alignment", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval=atoi(value)) > 0)) {
tmp_val = intval;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != intval) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_fr_alignment",
error_code);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Info_set(info, "romio_cb_fr_alignment", value);
fd->hints->cb_fr_alignment = intval;
}
/* for collective I/O, try to be smarter about when to do data sieving
* using a specific threshold for the datatype size/extent
* (percentage 0-100%) */
ADIOI_Info_get(users_info, "romio_cb_ds_threshold", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval=atoi(value)) > 0)) {
tmp_val = intval;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != intval) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_ds_threshold",
error_code);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Info_set(info, "romio_cb_ds_threshold", value);
fd->hints->cb_ds_threshold = intval;
}
ADIOI_Info_get(users_info, "romio_cb_alltoall", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
ADIOI_Info_set(info, "romio_cb_alltoall", value);
fd->hints->cb_read = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
ADIOI_Info_set(info, "romio_cb_alltoall", value);
fd->hints->cb_read = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(info, "romio_cb_alltoall", value);
fd->hints->cb_read = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->cb_alltoall;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->cb_alltoall) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_alltoall",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
#endif
/* new hints for enabling/disabling coll. buffering on
* reads/writes
*/
ADIOI_Info_get(users_info, "romio_cb_read", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
ADIOI_Info_set(info, "romio_cb_read", value);
fd->hints->cb_read = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
/* romio_cb_read overrides no_indep_rw */
ADIOI_Info_set(info, "romio_cb_read", value);
ADIOI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->cb_read = ADIOI_HINT_DISABLE;
fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(info, "romio_cb_read", value);
fd->hints->cb_read = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->cb_read;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->cb_read) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_read",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
ADIOI_Info_get(users_info, "romio_cb_write", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
ADIOI_Info_set(info, "romio_cb_write", value);
fd->hints->cb_write = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE"))
{
/* romio_cb_write overrides no_indep_rw, too */
ADIOI_Info_set(info, "romio_cb_write", value);
ADIOI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->cb_write = ADIOI_HINT_DISABLE;
fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") ||
!strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(info, "romio_cb_write", value);
fd->hints->cb_write = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->cb_write;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->cb_write) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_write",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
#if 0
/* bgl is not implementing file realms (ADIOI_IOStridedColl) ... */
/* enable/disable persistent file realms for collective I/O */
/* may want to check for no_indep_rdwr hint as well */
ADIOI_Info_get(users_info, "romio_cb_pfr", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
ADIOI_Info_set(info, "romio_cb_pfr", value);
fd->hints->cb_pfr = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
ADIOI_Info_set(info, "romio_cb_pfr", value);
fd->hints->cb_pfr = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(info, "romio_cb_pfr", value);
fd->hints->cb_pfr = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->cb_pfr;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->cb_pfr) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_pfr",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
/* file realm assignment types ADIOI_FR_AAR(0),
ADIOI_FR_FSZ(-1), ADIOI_FR_USR_REALMS(-2), all others specify
a regular fr size in bytes. probably not the best way... */
ADIOI_Info_get(users_info, "romio_cb_fr_type", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval=atoi(value)) >= -2)) {
tmp_val = intval;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != intval) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_cb_fr_type",
error_code);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Info_set(info, "romio_cb_fr_type", value);
fd->hints->cb_fr_type = intval;
}
#endif
/* new hint for specifying no indep. read/write will be performed */
ADIOI_Info_get(users_info, "romio_no_indep_rw", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "true") || !strcmp(value, "TRUE")) {
/* if 'no_indep_rw' set, also hint that we will do
* collective buffering: if we aren't doing independent io,
* then we have to do collective */
ADIOI_Info_set(info, "romio_no_indep_rw", value);
ADIOI_Info_set(info, "romio_cb_write", "enable");
ADIOI_Info_set(info, "romio_cb_read", "enable");
fd->hints->no_indep_rw = 1;
fd->hints->cb_read = 1;
fd->hints->cb_write = 1;
tmp_val = 1;
}
else if (!strcmp(value, "false") || !strcmp(value, "FALSE")) {
ADIOI_Info_set(info, "romio_no_indep_rw", value);
fd->hints->no_indep_rw = 0;
tmp_val = 0;
}
else {
/* default is above */
tmp_val = 0;
}
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
/* --BEGIN ERROR HANDLING-- */
if (tmp_val != fd->hints->no_indep_rw) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_no_indep_rw",
error_code);
return;
}
/* --END ERROR HANDLING-- */
}
/* new hints for enabling/disabling data sieving on
* reads/writes
*/
ADIOI_Info_get(users_info, "romio_ds_read", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
ADIOI_Info_set(info, "romio_ds_read", value);
fd->hints->ds_read = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
ADIOI_Info_set(info, "romio_ds_read", value);
fd->hints->ds_read = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(info, "romio_ds_read", value);
fd->hints->ds_read = ADIOI_HINT_AUTO;
}
/* otherwise ignore */
}
ADIOI_Info_get(users_info, "romio_ds_write", MPI_MAX_INFO_VAL, value,
&flag);
if (flag) {
if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
ADIOI_Info_set(info, "romio_ds_write", value);
fd->hints->ds_write = ADIOI_HINT_ENABLE;
}
else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
ADIOI_Info_set(info, "romio_ds_write", value);
fd->hints->ds_write = ADIOI_HINT_DISABLE;
}
else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(info, "romio_ds_write", value);
fd->hints->ds_write = ADIOI_HINT_AUTO;
}
/* otherwise ignore */
}
ADIOI_Info_get(users_info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval = atoi(value)) > 0)) {
ADIOI_Info_set(info, "ind_wr_buffer_size", value);
fd->hints->ind_wr_buffer_size = intval;
}
ADIOI_Info_get(users_info, "ind_rd_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval = atoi(value)) > 0)) {
ADIOI_Info_set(info, "ind_rd_buffer_size", value);
fd->hints->ind_rd_buffer_size = intval;
}
memset( value, 0, MPI_MAX_INFO_VAL+1 );
ADIOI_Info_get(users_info, "romio_min_fdomain_size", MPI_MAX_INFO_VAL,
value, &flag);
if ( flag && ((intval = atoi(value)) > 0) ) {
ADIOI_Info_set(info, "romio_min_fdomain_size", value);
fd->hints->min_fdomain_size = intval;
}
/* Now we use striping unit in common code so we should
process hints for it. */
ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
value, &flag);
if ( flag && ((intval = atoi(value)) > 0) ) {
ADIOI_Info_set(info, "striping_unit", value);
fd->hints->striping_unit = intval;
}
memset( value, 0, MPI_MAX_INFO_VAL+1 );
ADIOI_Info_get(users_info, ADIOI_BGL_NAGG_IN_PSET_HINT_NAME, MPI_MAX_INFO_VAL,
value, &flag);
if (flag && ((intval = atoi(value)) > 0)) {
did_anything = 1;
ADIOI_Info_set(info, ADIOI_BGL_NAGG_IN_PSET_HINT_NAME, value);
fd->hints->cb_nodes = intval;
}
}
/* associate CB aggregators to certain CNs in every involved PSET */
if (did_anything) {
ADIOI_BGL_gen_agg_ranklist(fd, fd->hints->cb_nodes);
}
/* ignore defered open hints and do not enable it for bluegene: need all
* processors in the open path so we can stat-and-broadcast the blocksize
*/
ADIOI_Info_set(info, "romio_no_indep_rw", "false");
fd->hints->no_indep_rw = 0;
fd->hints->deferred_open = 0;
/* BobC commented this out, but since hint processing runs on both bgl and
* bglockless, we need to keep DS writes enabled on gpfs and disabled on
* PVFS */
if (ADIO_Feature(fd, ADIO_DATA_SIEVING_WRITES) == 0) {
/* disable data sieving for fs that do not
support file locking */
ADIOI_Info_get(info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
/* get rid of this value if it is set */
ADIOI_Info_delete(info, "ind_wr_buffer_size");
}
/* note: leave ind_wr_buffer_size alone; used for other cases
* as well. -- Rob Ross, 04/22/2003
*/
ADIOI_Info_set(info, "romio_ds_write", "disable");
fd->hints->ds_write = ADIOI_HINT_DISABLE;
}
ADIOI_Free(value);
*error_code = MPI_SUCCESS;
}
void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
char *value;
int flag, stripe_val[3], str_factor = -1, str_unit=0, start_iodev=-1;
struct lov_user_md lum = { 0 };
int err, myrank, fd_sys, perm, amode, old_mask;
int int_val, tmp_val;
static char myname[] = "ADIOI_LUSTRE_SETINFO";
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
if ( (fd->info) == MPI_INFO_NULL) {
/* This must be part of the open call. can set striping parameters
if necessary. */
MPI_Info_create(&(fd->info));
ADIOI_Info_set(fd->info, "direct_read", "false");
ADIOI_Info_set(fd->info, "direct_write", "false");
fd->direct_read = fd->direct_write = 0;
/* initialize lustre hints */
ADIOI_Info_set(fd->info, "romio_lustre_co_ratio", "1");
fd->hints->fs_hints.lustre.co_ratio = 1;
ADIOI_Info_set(fd->info, "romio_lustre_coll_threshold", "0");
fd->hints->fs_hints.lustre.coll_threshold = 0;
ADIOI_Info_set(fd->info, "romio_lustre_ds_in_coll", "enable");
fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_ENABLE;
/* has user specified striping or server buffering parameters
and do they have the same value on all processes? */
if (users_info != MPI_INFO_NULL) {
/* striping information */
ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
value, &flag);
if (flag)
str_unit=atoi(value);
ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL,
value, &flag);
if (flag)
str_factor=atoi(value);
ADIOI_Info_get(users_info, "romio_lustre_start_iodevice",
MPI_MAX_INFO_VAL, value, &flag);
if (flag)
start_iodev=atoi(value);
/* direct read and write */
ADIOI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
ADIOI_Info_set(fd->info, "direct_read", "true");
fd->direct_read = 1;
}
ADIOI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
ADIOI_Info_set(fd->info, "direct_write", "true");
fd->direct_write = 1;
}
}
/* set striping information with ioctl */
MPI_Comm_rank(fd->comm, &myrank);
if (myrank == 0) {
stripe_val[0] = str_factor;
stripe_val[1] = str_unit;
stripe_val[2] = start_iodev;
}
MPI_Bcast(stripe_val, 3, MPI_INT, 0, fd->comm);
if (stripe_val[0] != str_factor
|| stripe_val[1] != str_unit
|| stripe_val[2] != start_iodev) {
FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: All keys"
"-striping_factor:striping_unit:start_iodevice "
"need to be identical across all processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
} else if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) {
/* if user has specified striping info, process 0 tries to set it */
if (!myrank) {
if (fd->perm == ADIO_PERM_NULL) {
old_mask = umask(022);
umask(old_mask);
perm = old_mask ^ 0666;
}
else perm = fd->perm;
amode = 0;
if (fd->access_mode & ADIO_CREATE)
amode = amode | O_CREAT;
if (fd->access_mode & ADIO_RDONLY)
amode = amode | O_RDONLY;
if (fd->access_mode & ADIO_WRONLY)
amode = amode | O_WRONLY;
if (fd->access_mode & ADIO_RDWR)
amode = amode | O_RDWR;
if (fd->access_mode & ADIO_EXCL)
amode = amode | O_EXCL;
/* we need to create file so ensure this is set */
amode = amode | O_LOV_DELAY_CREATE | O_CREAT;
fd_sys = open(fd->filename, amode, perm);
if (fd_sys == -1) {
if (errno != EEXIST)
fprintf(stderr,
"Failure to open file %s %d %d\n",strerror(errno), amode, perm);
} else {
lum.lmm_magic = LOV_USER_MAGIC;
lum.lmm_pattern = 0;
lum.lmm_stripe_size = str_unit;
lum.lmm_stripe_count = str_factor;
lum.lmm_stripe_offset = start_iodev;
err = ioctl(fd_sys, LL_IOC_LOV_SETSTRIPE, &lum);
if (err == -1 && errno != EEXIST) {
fprintf(stderr, "Failure to set stripe info %s \n", strerror(errno));
}
close(fd_sys);
}
} /* End of striping parameters validation */
}
MPI_Barrier(fd->comm);
}
/* get other hint */
if (users_info != MPI_INFO_NULL) {
/* CO: IO Clients/OST,
* to keep the load balancing between clients and OSTs */
ADIOI_Info_get(users_info, "romio_lustre_co_ratio", MPI_MAX_INFO_VAL, value,
&flag);
if (flag && (int_val = atoi(value)) > 0) {
tmp_val = int_val;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
if (tmp_val != int_val) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_lustre_co_ratio",
error_code);
ADIOI_Free(value);
return;
}
ADIOI_Info_set(fd->info, "romio_lustre_co_ratio", value);
fd->hints->fs_hints.lustre.co_ratio = atoi(value);
}
/* coll_threshold:
* if the req size is bigger than this, collective IO may not be performed.
*/
ADIOI_Info_get(users_info, "romio_lustre_coll_threshold", MPI_MAX_INFO_VAL, value,
&flag);
if (flag && (int_val = atoi(value)) > 0) {
tmp_val = int_val;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
if (tmp_val != int_val) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_lustre_coll_threshold",
error_code);
ADIOI_Free(value);
return;
}
ADIOI_Info_set(fd->info, "romio_lustre_coll_threshold", value);
fd->hints->fs_hints.lustre.coll_threshold = atoi(value);
}
/* ds_in_coll: disable data sieving in collective IO */
ADIOI_Info_get(users_info, "romio_lustre_ds_in_coll", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && (!strcmp(value, "disable") ||
!strcmp(value, "DISABLE"))) {
tmp_val = int_val = 2;
MPI_Bcast(&tmp_val, 2, MPI_INT, 0, fd->comm);
if (tmp_val != int_val) {
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_lustre_ds_in_coll",
error_code);
ADIOI_Free(value);
return;
}
ADIOI_Info_set(fd->info, "romio_lustre_ds_in_coll", "disable");
fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_DISABLE;
}
}
/* set the values for collective I/O and data sieving parameters */
ADIOI_GEN_SetInfo(fd, users_info, error_code);
if (ADIOI_Direct_read) fd->direct_read = 1;
if (ADIOI_Direct_write) fd->direct_write = 1;
ADIOI_Free(value);
*error_code = MPI_SUCCESS;
}
void ADIOI_PVFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
char *value;
int flag, tmp_val, str_factor=-1, str_unit=-1, start_iodev=-1;
static char myname[] = "ADIOI_PVFS_SETINFO";
if ((fd->info) == MPI_INFO_NULL) {
/* This must be part of the open call. can set striping parameters
if necessary. */
MPI_Info_create(&(fd->info));
ADIOI_Info_set(fd->info, "romio_pvfs_listio_read", "disable");
ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", "disable");
fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_DISABLE;
fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_DISABLE;
/* has user specified any pvfs-specific hints (striping params, listio)
and do they have the same value on all processes? */
if (users_info != MPI_INFO_NULL) {
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
str_factor=atoi(value);
tmp_val = str_factor;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
if (tmp_val != str_factor) {
/* --BEGIN ERROR HANDLING-- */
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"striping_factor",
error_code);
return;
/* --END ERROR HANDLING-- */
}
else ADIOI_Info_set(fd->info, "striping_factor", value);
}
ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
str_unit=atoi(value);
tmp_val = str_unit;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
if (tmp_val != str_unit) {
/* --BEGIN ERROR HANDLING-- */
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"striping_unit",
error_code);
return;
/* --END ERROR HANDLING-- */
}
else ADIOI_Info_set(fd->info, "striping_unit", value);
}
ADIOI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
start_iodev=atoi(value);
tmp_val = start_iodev;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
if (tmp_val != start_iodev) {
/* --BEGIN ERROR HANDLING-- */
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"start_iodevice",
error_code);
return;
/* --END ERROR HANDLING-- */
}
else ADIOI_Info_set(fd->info, "start_iodevice", value);
}
ADIOI_Info_get(users_info, "romio_pvfs_listio_read",
MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs_listio_read", value);
fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_ENABLE;
} else if ( !strcmp(value, "disable") || !strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info , "romio_pvfs_listio_read", value);
fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_DISABLE;
}
else if ( !strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(fd->info, "romio_pvfs_listio_read", value);
fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->fs_hints.pvfs.listio_read;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
if (tmp_val != fd->hints->fs_hints.pvfs.listio_read) {
/* --BEGIN ERROR HANDLING-- */
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_pvfs_listio_read",
error_code);
return;
/* --END ERROR HANDLING-- */
}
}
ADIOI_Info_get(users_info, "romio_pvfs_listio_write", MPI_MAX_INFO_VAL,
value, &flag);
if (flag) {
if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", value);
fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_ENABLE;
} else if ( !strcmp(value, "disable") || !strcmp(value, "DISABLE"))
{
ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", value);
fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_DISABLE;
}
else if ( !strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
{
ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", value);
fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_AUTO;
}
tmp_val = fd->hints->fs_hints.pvfs.listio_write;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
if (tmp_val != fd->hints->fs_hints.pvfs.listio_write) {
/* --BEGIN ERROR HANDLING-- */
MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
"romio_pvfs_listio_write",
error_code);
return;
/* --END ERROR HANDLING-- */
}
}
ADIOI_Free(value);
}
}
/* set the values for collective I/O and data sieving parameters */
ADIOI_GEN_SetInfo(fd, users_info, error_code);
*error_code = MPI_SUCCESS;
}
