int mca_io_ompio_file_iwrite_at_all (ompi_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf, int count, struct ompi_datatype_t *datatype, ompi_request_t **request) { int ret = OMPI_SUCCESS; mca_io_ompio_data_t *data; data = (mca_io_ompio_data_t *) fh->f_io_selected_data; ret = mca_common_ompio_file_iwrite_at_all ( &data->ompio_fh, offset, buf, count, datatype, request ); return ret; }
int mca_io_ompio_file_write_at_all_begin (ompi_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf, int count, struct ompi_datatype_t *datatype) { int ret = OMPI_SUCCESS; mca_io_ompio_data_t *data=NULL; mca_io_ompio_file_t *fp=NULL; data = (mca_io_ompio_data_t *) fh->f_io_selected_data; fp = &data->ompio_fh; if ( true == fp->f_split_coll_in_use ) { printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n"); return MPI_ERR_REQUEST; } ret = mca_common_ompio_file_iwrite_at_all ( fp, offset, buf, count, datatype, &fp->f_split_coll_req ); fp->f_split_coll_in_use = true; return ret; }
int mca_sharedfp_addproc_write_ordered_begin(mca_io_ompio_file_t *fh, const void *buf, int count, struct ompi_datatype_t *datatype) { int ret = OMPI_SUCCESS; OMPI_MPI_OFFSET_TYPE offset = 0, offsetReceived = 0; long sendBuff = 0; long *buff=NULL; long offsetBuff; long bytesRequested = 0; int recvcnt = 1, sendcnt = 1; size_t numofBytes; int rank, size, i; struct mca_sharedfp_base_data_t *sh = NULL; if ( NULL == fh->f_sharedfp_data){ opal_output(0, "sharedfp_addproc_write_ordered_begin:" " shared file pointer structure not initialized correctly\n"); return OMPI_ERROR; } if ( true == fh->f_split_coll_in_use ) { opal_output(0, "Only one split collective I/O operation allowed per file handle " "at any given point in time!\n"); return MPI_ERR_REQUEST; } /*Retrieve the shared file pointer structure*/ sh = fh->f_sharedfp_data; /* Calculate the number of bytes to write*/ opal_datatype_type_size ( &datatype->super, &numofBytes); sendBuff = count * numofBytes; /* Get the ranks in the communicator */ rank = ompi_comm_rank ( sh->comm ); size = ompi_comm_size ( sh->comm ); if ( 0 == rank ) { buff = (long*)malloc(sizeof(OMPI_MPI_OFFSET_TYPE) * size); if ( NULL == buff ) return OMPI_ERR_OUT_OF_RESOURCE; } ret = sh->comm->c_coll.coll_gather ( &sendBuff, sendcnt, OMPI_OFFSET_DATATYPE, buff, recvcnt, OMPI_OFFSET_DATATYPE, 0, sh->comm, sh->comm->c_coll.coll_gather_module); if( OMPI_SUCCESS != ret ){ goto exit; } /* All the counts are present now in the recvBuff. The size of recvBuff is sizeof_newComm */ if ( 0 == rank ) { for (i = 0; i < size ; i ++) { bytesRequested += buff[i]; if ( mca_sharedfp_addproc_verbose ){ opal_output(ompi_sharedfp_base_framework.framework_output, "sharedfp_addproc_write_ordered_begin: Bytes requested are %ld\n", bytesRequested); } } /* Request the offset to write bytesRequested bytes ** only the root process needs to do the request, ** since the root process will then tell the other ** processes at what offset they should write their ** share of the data. */ ret = mca_sharedfp_addproc_request_position(sh,bytesRequested,&offsetReceived); if( OMPI_SUCCESS != ret ){ goto exit; } if ( mca_sharedfp_addproc_verbose ){ opal_output(ompi_sharedfp_base_framework.framework_output, "sharedfp_addproc_write_ordered_begin: Offset received is %lld\n", offsetReceived); } buff[0] += offsetReceived; for (i = 1 ; i < size; i++) { buff[i] += buff[i-1]; } } /* Scatter the results to the other processes*/ ret = sh->comm->c_coll.coll_scatter ( buff, sendcnt, OMPI_OFFSET_DATATYPE, &offsetBuff, recvcnt, OMPI_OFFSET_DATATYPE, 0, sh->comm, sh->comm->c_coll.coll_scatter_module ); if( OMPI_SUCCESS != ret ){ goto exit; } /*Each process now has its own individual offset in recvBUFF*/ offset = offsetBuff - sendBuff; offset /= sh->sharedfh->f_etype_size; if ( mca_sharedfp_addproc_verbose ){ opal_output(ompi_sharedfp_base_framework.framework_output, "sharedfp_addproc_write_ordered: Offset returned is %lld\n",offset); } /* write to the file */ ret = mca_common_ompio_file_iwrite_at_all(sh->sharedfh,offset,buf,count,datatype,&fh->f_split_coll_req); fh->f_split_coll_in_use = true; exit: if ( NULL != buff ) { free ( buff ); } return ret; }
int mca_sharedfp_individual_write_ordered_begin(mca_io_ompio_file_t *fh, const void *buf, int count, struct ompi_datatype_t *datatype) { int ret = OMPI_SUCCESS; int size = 0, rank = 0; int i = 0; size_t numofbytes = 0; size_t totalbytes = 0; OMPI_MPI_OFFSET_TYPE *offbuff=NULL; OMPI_MPI_OFFSET_TYPE global_offset = 0; OMPI_MPI_OFFSET_TYPE prev_offset = 0; OMPI_MPI_OFFSET_TYPE temp = 0, offset = 0; mca_sharedfp_individual_header_record *headnode = NULL; struct mca_sharedfp_base_data_t *sh = NULL; mca_sharedfp_base_module_t * shared_fp_base_module = NULL; if(fh->f_sharedfp_data==NULL){ if ( mca_sharedfp_individual_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "sharedfp_individual_write_ordered_begin - opening the shared file pointer\n"); } shared_fp_base_module = fh->f_sharedfp; ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm, fh->f_filename, fh->f_amode, fh->f_info, fh); if ( OMPI_SUCCESS != ret ) { opal_output(0,"sharedfp_individual_write_ordered_begin - error opening the shared file pointer\n"); return ret; } } if ( true == fh->f_split_coll_in_use ) { opal_output(0, "Only one split collective I/O operation allowed per file handle at any given point in time!\n"); return MPI_ERR_REQUEST; } /*Retrieve the sharedfp data structures*/ sh = fh->f_sharedfp_data; rank = ompi_comm_rank ( sh->comm ); size = ompi_comm_size ( sh->comm ); /* Calculate the number of bytes of data that needs to be written*/ opal_datatype_type_size ( &datatype->super, &numofbytes); totalbytes = count * numofbytes; headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data; if ( NULL == headnode) { opal_output (0, "sharedfp_individual_write_ordered_begin: headnode is NULL but file is open\n"); return OMPI_ERROR; } /* Data from all the metadata is combined and written to the main file */ ret = mca_sharedfp_individual_collaborate_data ( sh ); if ( OMPI_SUCCESS != ret) { return ret; } if ( 0 == rank ) { offbuff = (OMPI_MPI_OFFSET_TYPE *)malloc ( sizeof(OMPI_MPI_OFFSET_TYPE) * size); if (NULL == offbuff ) { return OMPI_ERR_OUT_OF_RESOURCE; } } /*collect the total bytes to be written*/ sh->comm->c_coll->coll_gather ( &totalbytes, 1, OMPI_OFFSET_DATATYPE, offbuff, 1, OMPI_OFFSET_DATATYPE, 0, sh->comm, sh->comm->c_coll->coll_gather_module ); if ( 0 == rank ) { prev_offset = offbuff[0]; offbuff[0] = sh->global_offset; for (i = 1; i < size ; i++){ temp = offbuff[i]; offbuff[i] = offbuff[i - 1] + prev_offset; prev_offset = temp; } for (i = 0; i < size; i++){ global_offset = offbuff[size - 1] + prev_offset; } } /* Scatter the results to the other processes */ ret = sh->comm->c_coll->coll_scatter ( offbuff, 1, OMPI_OFFSET_DATATYPE, &offset, 1, OMPI_OFFSET_DATATYPE, 0, sh->comm, sh->comm->c_coll->coll_scatter_module ); if ( OMPI_SUCCESS != ret ) { opal_output(0,"sharedfp_individual_write_ordered_begin: Error in scattering offsets \n"); goto exit; } ret = sh->comm->c_coll->coll_bcast ( &global_offset, 1, OMPI_OFFSET_DATATYPE, 0, sh->comm, sh->comm->c_coll->coll_bcast_module ); if ( OMPI_SUCCESS != ret ) { opal_output(0,"sharedfp_individual_write_ordered_begin: Error while bcasting global offset \n"); goto exit; } sh->global_offset = global_offset; /*use file_write_at_all to ensure the order*/ ret = mca_common_ompio_file_iwrite_at_all(sh->sharedfh,offset, buf,count,datatype, &fh->f_split_coll_req); fh->f_split_coll_in_use = true; if ( OMPI_SUCCESS != ret ) { opal_output(0,"sharedfp_individual_write_ordered_begin: Error while writing the datafile \n"); } exit: if ( NULL != offbuff ) { free ( offbuff); } return ret; }