int mca_io_ompio_file_write (ompi_file_t *fp,
const void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS;
mca_io_ompio_data_t *data;
mca_io_ompio_file_t *fh;
data = (mca_io_ompio_data_t *) fp->f_io_selected_data;
fh = &data->ompio_fh;
ret = mca_common_ompio_file_write(fh,buf,count,datatype,status);
return ret;
}
int mca_common_ompio_file_write_at (ompio_file_t *fh,
OMPI_MPI_OFFSET_TYPE offset,
const void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS;
OMPI_MPI_OFFSET_TYPE prev_offset;
mca_common_ompio_file_get_position (fh, &prev_offset );
mca_common_ompio_set_explicit_offset (fh, offset);
ret = mca_common_ompio_file_write (fh,
buf,
count,
datatype,
status);
// An explicit offset file operation is not suppsed to modify
// the internal file pointer. So reset the pointer
// to the previous value
mca_common_ompio_set_explicit_offset (fh, prev_offset );
return ret;
}
int mca_io_ompio_file_preallocate (ompi_file_t *fh,
OMPI_MPI_OFFSET_TYPE diskspace)
{
int ret = OMPI_SUCCESS, cycles, i;
OMPI_MPI_OFFSET_TYPE tmp, current_size, size, written, len;
mca_io_ompio_data_t *data;
char *buf = NULL;
ompi_status_public_t *status = NULL;
data = (mca_io_ompio_data_t *) fh->f_io_selected_data;
OPAL_THREAD_LOCK(&fh->f_lock);
tmp = diskspace;
ret = data->ompio_fh.f_comm->c_coll->coll_bcast (&tmp,
1,
OMPI_OFFSET_DATATYPE,
OMPIO_ROOT,
data->ompio_fh.f_comm,
data->ompio_fh.f_comm->c_coll->coll_bcast_module);
if ( OMPI_SUCCESS != ret ) {
OPAL_THREAD_UNLOCK(&fh->f_lock);
return OMPI_ERROR;
}
if (tmp != diskspace) {
OPAL_THREAD_UNLOCK(&fh->f_lock);
return OMPI_ERROR;
}
ret = data->ompio_fh.f_fs->fs_file_get_size (&data->ompio_fh,
¤t_size);
if ( OMPI_SUCCESS != ret ) {
OPAL_THREAD_UNLOCK(&fh->f_lock);
return OMPI_ERROR;
}
if ( current_size > diskspace ) {
OPAL_THREAD_UNLOCK(&fh->f_lock);
return OMPI_SUCCESS;
}
/* ROMIO explanation
On file systems with no preallocation function, we have to
explicitly write to allocate space. Since there could be holes in the file,
we need to read up to the current file size, write it back,
and then write beyond that depending on how much
preallocation is needed.
*/
if (OMPIO_ROOT == data->ompio_fh.f_rank) {
OMPI_MPI_OFFSET_TYPE prev_offset;
mca_common_ompio_file_get_position (&data->ompio_fh, &prev_offset );
size = diskspace;
if (size > current_size) {
size = current_size;
}
cycles = (size + OMPIO_PREALLOC_MAX_BUF_SIZE - 1)/
OMPIO_PREALLOC_MAX_BUF_SIZE;
buf = (char *) malloc (OMPIO_PREALLOC_MAX_BUF_SIZE);
if (NULL == buf) {
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
written = 0;
for (i=0; i<cycles; i++) {
len = OMPIO_PREALLOC_MAX_BUF_SIZE;
if (len > size-written) {
len = size - written;
}
ret = mca_common_ompio_file_read (&data->ompio_fh, buf, len, MPI_BYTE, status);
if (ret != OMPI_SUCCESS) {
goto exit;
}
ret = mca_common_ompio_file_write (&data->ompio_fh, buf, len, MPI_BYTE, status);
if (ret != OMPI_SUCCESS) {
goto exit;
}
written += len;
}
if (diskspace > current_size) {
memset(buf, 0, OMPIO_PREALLOC_MAX_BUF_SIZE);
size = diskspace - current_size;
cycles = (size + OMPIO_PREALLOC_MAX_BUF_SIZE - 1) /
OMPIO_PREALLOC_MAX_BUF_SIZE;
for (i=0; i<cycles; i++) {
len = OMPIO_PREALLOC_MAX_BUF_SIZE;
if (len > diskspace-written) {
len = diskspace - written;
}
ret = mca_common_ompio_file_write (&data->ompio_fh, buf, len, MPI_BYTE, status);
if (ret != OMPI_SUCCESS) {
goto exit;
}
written += len;
}
}
// This operation should not affect file pointer position.
mca_common_ompio_set_explicit_offset ( &data->ompio_fh, prev_offset);
}
exit:
free ( buf );
fh->f_comm->c_coll->coll_bcast ( &ret, 1, MPI_INT, OMPIO_ROOT, fh->f_comm,
fh->f_comm->c_coll->coll_bcast_module);
if ( diskspace > current_size ) {
data->ompio_fh.f_fs->fs_file_set_size (&data->ompio_fh, diskspace);
}
OPAL_THREAD_UNLOCK(&fh->f_lock);
return ret;
}
int mca_common_ompio_file_iwrite (ompio_file_t *fh,
const void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_request_t **request)
{
int ret = OMPI_SUCCESS;
mca_ompio_request_t *ompio_req=NULL;
size_t spc=0;
mca_common_ompio_request_alloc ( &ompio_req, MCA_OMPIO_REQUEST_WRITE);
if ( 0 == count ) {
ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
ompio_req->req_ompi.req_status._ucount = 0;
ompi_request_complete (&ompio_req->req_ompi, false);
*request = (ompi_request_t *) ompio_req;
return OMPI_SUCCESS;
}
if ( NULL != fh->f_fbtl->fbtl_ipwritev ) {
/* This fbtl has support for non-blocking operations */
uint32_t iov_count = 0;
struct iovec *decoded_iov = NULL;
size_t max_data = 0;
size_t total_bytes_written =0;
int i = 0; /* index into the decoded iovec of the buffer */
int j = 0; /* index into the file vie iovec */
mca_common_ompio_decode_datatype (fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
j = fh->f_index_in_file_view;
/* Non blocking operations have to occur in a single cycle */
mca_common_ompio_build_io_array ( fh,
0, // index of current cycle iteration
1, // number of cycles
max_data, // setting bytes_per_cycle to max_data
max_data,
iov_count,
decoded_iov,
&i,
&j,
&total_bytes_written,
&spc);
if (fh->f_num_of_io_entries) {
fh->f_fbtl->fbtl_ipwritev (fh, (ompi_request_t *) ompio_req);
}
mca_common_ompio_register_progress ();
fh->f_num_of_io_entries = 0;
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
if (NULL != decoded_iov) {
free (decoded_iov);
decoded_iov = NULL;
}
}
else {
// This fbtl does not support non-blocking write operations
ompi_status_public_t status;
ret = mca_common_ompio_file_write(fh,buf,count,datatype, &status);
ompio_req->req_ompi.req_status.MPI_ERROR = ret;
ompio_req->req_ompi.req_status._ucount = status._ucount;
ompi_request_complete (&ompio_req->req_ompi, false);
}
*request = (ompi_request_t *) ompio_req;
return ret;
}
int mca_common_ompio_file_iwrite (ompio_file_t *fh,
const void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_request_t **request)
{
int ret = OMPI_SUCCESS;
mca_ompio_request_t *ompio_req=NULL;
size_t spc=0;
if (fh->f_amode & MPI_MODE_RDONLY){
// opal_output(10, "Improper use of FILE Mode, Using RDONLY for write!\n");
ret = MPI_ERR_READ_ONLY;
return ret;
}
mca_common_ompio_request_alloc ( &ompio_req, MCA_OMPIO_REQUEST_WRITE);
if ( 0 == count ) {
ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
ompio_req->req_ompi.req_status._ucount = 0;
ompi_request_complete (&ompio_req->req_ompi, false);
*request = (ompi_request_t *) ompio_req;
return OMPI_SUCCESS;
}
if ( NULL != fh->f_fbtl->fbtl_ipwritev ) {
/* This fbtl has support for non-blocking operations */
uint32_t iov_count = 0;
struct iovec *decoded_iov = NULL;
size_t max_data = 0;
size_t total_bytes_written =0;
int i = 0; /* index into the decoded iovec of the buffer */
int j = 0; /* index into the file vie iovec */
#if OPAL_CUDA_SUPPORT
int is_gpu, is_managed;
mca_common_ompio_check_gpu_buf ( fh, buf, &is_gpu, &is_managed);
if ( is_gpu && !is_managed ) {
size_t pos=0;
char *tbuf=NULL;
opal_convertor_t convertor;
OMPIO_CUDA_PREPARE_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);
opal_convertor_pack (&convertor, decoded_iov, &iov_count, &pos );
opal_convertor_cleanup (&convertor);
ompio_req->req_tbuf = tbuf;
ompio_req->req_size = max_data;
}
else {
mca_common_ompio_decode_datatype (fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
}
#else
mca_common_ompio_decode_datatype (fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
#endif
if ( 0 < max_data && 0 == fh->f_iov_count ) {
ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
ompio_req->req_ompi.req_status._ucount = 0;
ompi_request_complete (&ompio_req->req_ompi, false);
*request = (ompi_request_t *) ompio_req;
return OMPI_SUCCESS;
}
j = fh->f_index_in_file_view;
/* Non blocking operations have to occur in a single cycle */
mca_common_ompio_build_io_array ( fh,
0, // index of current cycle iteration
1, // number of cycles
max_data, // setting bytes_per_cycle to max_data
max_data,
iov_count,
decoded_iov,
&i,
&j,
&total_bytes_written,
&spc);
if (fh->f_num_of_io_entries) {
fh->f_fbtl->fbtl_ipwritev (fh, (ompi_request_t *) ompio_req);
}
mca_common_ompio_register_progress ();
fh->f_num_of_io_entries = 0;
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
if (NULL != decoded_iov) {
free (decoded_iov);
decoded_iov = NULL;
}
}
else {
// This fbtl does not support non-blocking write operations
ompi_status_public_t status;
ret = mca_common_ompio_file_write(fh,buf,count,datatype, &status);
ompio_req->req_ompi.req_status.MPI_ERROR = ret;
ompio_req->req_ompi.req_status._ucount = status._ucount;
ompi_request_complete (&ompio_req->req_ompi, false);
}
*request = (ompi_request_t *) ompio_req;
return ret;
}
