int mca_sharedfp_individual_write (mca_io_ompio_file_t *fh,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS;
size_t numofbytes = 0;
size_t totalbytes = 0;
mca_sharedfp_individual_header_record *headnode = NULL;
struct mca_sharedfp_base_data_t *sh = NULL;
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
if ( NULL == fh->f_sharedfp_data ) {
if ( mca_sharedfp_individual_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,"sharedfp_individual_write: opening the shared file pointer file\n");
}
shared_fp_base_module = fh->f_sharedfp;
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
fh->f_filename,
fh->f_amode,
fh->f_info,
fh);
if (ret != OMPI_SUCCESS) {
opal_output(0,"sharedfp_individual_write - error opening the shared file pointer\n");
return ret;
}
}
/* Calculate the number of bytes of data that need to be written*/
opal_datatype_type_size ( &datatype->super, &numofbytes);
totalbytes = count * numofbytes;
/*Retrieve data structure for shared file pointer operations*/
sh = fh->f_sharedfp_data;
headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data;
if (headnode) {
/*Insert metadata record into a queue*/
mca_sharedfp_individual_insert_metadata(OMPI_FILE_WRITE_SHARED, totalbytes, sh);
/*Write the data into individual file*/
ret = ompio_io_ompio_file_write_at ( headnode->datafilehandle,
headnode->datafile_offset,
buf, count, datatype, status);
if ( OMPI_SUCCESS != ret ) {
opal_output(0,"mca_sharedfp_individual_write: Error while writing the datafile \n");
return -1;
}
/* Update the datafileoffset*/
headnode->datafile_offset = headnode->datafile_offset + totalbytes;
}
return ret;
}
int mca_sharedfp_sm_write (mca_io_ompio_file_t *fh,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS;
OMPI_MPI_OFFSET_TYPE offset = 0;
long bytesRequested = 0;
size_t numofBytes;
struct mca_sharedfp_base_data_t *sh = NULL;
mca_sharedfp_base_module_t * shared_fp_base_module = NULL;
if( NULL == fh->f_sharedfp_data ){
if ( mca_sharedfp_sm_verbose ) {
printf("sharedfp_sm_write: opening the shared file pointer\n");
}
shared_fp_base_module = fh->f_sharedfp;
ret = shared_fp_base_module->sharedfp_file_open(fh->f_comm,
fh->f_filename,
fh->f_amode,
fh->f_info,
fh);
if ( OMPI_SUCCESS != ret ) {
opal_output(0,"sharedfp_sm_write - error opening the shared file pointer\n");
return ret;
}
}
/* Calculate the number of bytes to write*/
opal_datatype_type_size ( &datatype->super, &numofBytes);
bytesRequested = count * numofBytes;
/*Retrieve the shared file data struct*/
sh = fh->f_sharedfp_data;
if ( mca_sharedfp_sm_verbose ) {
printf("sharedfp_sm_write: Requested is %ld\n",bytesRequested);
}
/*Request the offset to write bytesRequested bytes*/
ret = mca_sharedfp_sm_request_position(sh,bytesRequested,&offset);
if ( -1 != ret ) {
if ( mca_sharedfp_sm_verbose ) {
printf("sharedfp_sm_write: fset received is %lld\n",offset);
}
/* Write to the file*/
ret = ompio_io_ompio_file_write_at(sh->sharedfh,offset,buf,count,datatype,status);
}
return ret;
}
int mca_sharedfp_individual_write_metadata_file(struct mca_sharedfp_base_data_t *sh)
{
mca_sharedfp_individual_metadata_node *current = NULL;
struct mca_sharedfp_individual_record2 buff;
mca_sharedfp_individual_header_record *headnode = NULL;
int ret=OMPI_SUCCESS;
MPI_Status status;
headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data;
if (headnode->numofrecordsonfile == 0)
headnode->metadatafile_offset = headnode->metafile_start_offset;
current = headnode->next;
while (current != NULL) {
/*Read from the linked list*/
buff.recordid = current->recordid;
buff.timestamp = current->timestamp;
buff.localposition = current->localposition;
buff.recordlength = current->recordlength;
if ( mca_sharedfp_individual_verbose ) {
printf("sharedfp_individual_write_metadata_file: Buff recordid %ld\n",buff.recordid);
printf("sharedfp_individual_write_metadata_file: Buff timestamp %f\n", buff.timestamp);
printf("sharedfp_individual_write_metadata_file: Buff localposition %lld\n",buff.localposition);
printf("sharedfp_individual_write_metadata_file: Buff recordlength %ld\n",buff.recordlength);
printf("sharedfp_individual_write_metadata_file: Size of buff %ld\n",sizeof(buff));
}
headnode->next = current->next;
free(current);
current = (headnode)->next;
/*Write to the metadata file*/
ret = ompio_io_ompio_file_write_at ( (headnode)->metadatafilehandle,
(headnode)->metadatafile_offset,
&buff,32, MPI_BYTE, &status);
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
(headnode)->numofrecordsonfile = headnode->numofrecordsonfile + 1;
(headnode)->metadatafile_offset = (headnode)->metadatafile_offset + sizeof(buff);
}
/*All records are being read from the linked list and written to the file*/
(headnode)->numofrecords = 0;
exit:
return ret;
}
int mca_io_ompio_file_write_at (ompi_file_t *fh,
OMPI_MPI_OFFSET_TYPE offset,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS;
mca_io_ompio_data_t *data;
data = (mca_io_ompio_data_t *) fh->f_io_selected_data;
ret = ompio_io_ompio_file_write_at (&data->ompio_fh, offset,buf,count,datatype,status);
return ret;
}
int mca_sharedfp_addproc_write (mca_io_ompio_file_t *fh,
const void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS;
OMPI_MPI_OFFSET_TYPE offset = 0;
long bytesRequested = 0;
size_t numofBytes;
struct mca_sharedfp_base_data_t *sh = NULL;
if(NULL == fh->f_sharedfp_data){
opal_output(0, "sharedfp_addproc_write: shared file pointer structure not initialized correctly\n");
return OMPI_ERROR;
}
/* Calculate the number of bytes to write*/
opal_datatype_type_size ( &datatype->super, &numofBytes);
bytesRequested = count * numofBytes;
/*Retrieve the shared file data structure */
sh = fh->f_sharedfp_data;
if ( mca_sharedfp_addproc_verbose ){
opal_output(ompi_sharedfp_base_framework.framework_output,
"sharedfp_addproc_write: sharedfp_addproc_write: Bytes Requested is %ld\n",
bytesRequested);
}
/*Request the offset to write bytesRequested bytes*/
ret = mca_sharedfp_addproc_request_position( sh, bytesRequested, &offset);
offset /= sh->sharedfh->f_etype_size;
if ( OMPI_SUCCESS == ret ) {
if ( mca_sharedfp_addproc_verbose ){
opal_output(ompi_sharedfp_base_framework.framework_output,
"sharedfp_addproc_write: Offset received is %lld\n",offset);
}
/* Write to the file */
ret = ompio_io_ompio_file_write_at(sh->sharedfh,offset,buf,count,datatype,status);
}
return ret;
}
int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh)
{
int ret = OMPI_SUCCESS;
mca_sharedfp_individual_header_record *headnode = NULL;
char *buff=NULL;
MPI_Comm comm;
int rank, size;
int nodesoneachprocess = 0;
int idx = 0,i = 0;
double *timestampbuff = NULL;
OMPI_MPI_OFFSET_TYPE *offsetbuff = NULL;
int *countbuff = NULL;
int *displ = NULL;
double *ind_ts = NULL;
long *ind_recordlength = NULL;
OMPI_MPI_OFFSET_TYPE *local_off = NULL;
int totalnodes = 0;
ompi_status_public_t status;
comm = sh->comm;
rank = ompi_comm_rank ( comm );
size = ompi_comm_size ( comm );
headnode = (mca_sharedfp_individual_header_record*)sh->selected_module_data;
if ( NULL == headnode) {
opal_output (1, "sharedfp_individual_collaborate_data: headnode is NULL but file is open\n");
return OMPI_ERROR;
}
/* Number of nodes on each process is the sum of records
* on file and records in the linked list
*/
nodesoneachprocess = headnode->numofrecordsonfile + headnode->numofrecords;
if ( mca_sharedfp_individual_verbose ) {
printf("Nodes of each process = %d\n",nodesoneachprocess);
}
countbuff = (int*)malloc(size * sizeof(int));
if ( NULL == countbuff ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
displ = (int*)malloc(sizeof(int) * size);
if ( NULL == displ ) {
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
/* Each process counts the number of nodes
* in its linked list for which global offset */
ret = mca_sharedfp_individual_get_timestamps_and_reclengths ( &ind_ts, &ind_recordlength,
&local_off, sh );
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
comm->c_coll.coll_allgather ( &nodesoneachprocess, 1, MPI_INT,
countbuff, 1, MPI_INT, comm,
comm->c_coll.coll_allgather_module );
if ( mca_sharedfp_individual_verbose) {
for (i = 0; i < size ; i++) {
printf("sharedfp_individual_collaborate_data: Countbuff[%d] = %d\n", i, countbuff[i]);
}
}
if ( nodesoneachprocess == 0) {
ind_ts[0] = 0;
ind_recordlength[0] = 0;
local_off[0] = 0;
}
for(i = 0; i < size; i++) {
displ[i] = totalnodes;
if ( mca_sharedfp_individual_verbose ) {
printf("sharedfp_individual_collaborate_data: displ[%d] = %d\n",i,displ[i]);
}
totalnodes = totalnodes + countbuff[i];
}
if (totalnodes <= 0 ) {
goto exit;
}
ret = mca_sharedfp_individual_create_buff ( ×tampbuff, &offsetbuff, totalnodes, size);
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
ret = comm->c_coll.coll_allgatherv ( ind_ts, countbuff[rank], MPI_DOUBLE,
timestampbuff, countbuff, displ, MPI_DOUBLE,
comm, comm->c_coll.coll_allgatherv_module );
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
ret = comm->c_coll.coll_allgatherv ( ind_recordlength, countbuff[rank], OMPI_OFFSET_DATATYPE,
offsetbuff, countbuff, displ, OMPI_OFFSET_DATATYPE,
comm, comm->c_coll.coll_allgatherv_module );
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
ret = mca_sharedfp_individual_sort_timestamps(×tampbuff, &offsetbuff,totalnodes);
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
sh->global_offset = mca_sharedfp_individual_assign_globaloffset ( &offsetbuff, totalnodes, sh);
buff = (char * ) malloc( ind_recordlength[0] * 1.2 );
if ( NULL == buff ) {
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (i = 0; i < nodesoneachprocess ; i++) {
/*Read from the local data file*/
ompio_io_ompio_file_read_at ( headnode->datafilehandle,
local_off[i], buff, ind_recordlength[i],
MPI_BYTE, &status);
idx = mca_sharedfp_individual_getoffset(ind_ts[i],timestampbuff,totalnodes);
if ( mca_sharedfp_individual_verbose ) {
printf("sharedfp_individual_collaborate_data: Process %d writing %ld bytes to main file \n",
rank,ind_recordlength[i]);
}
/*Write into main data file*/
ompio_io_ompio_file_write_at( sh->sharedfh, offsetbuff[idx], buff,
ind_recordlength[i], MPI_BYTE, &status);
}
exit:
if ( NULL != countbuff ) {
free ( countbuff );
}
if ( NULL != displ ) {
free ( displ );
}
if( NULL != timestampbuff ){
free ( timestampbuff );
}
if ( NULL != offsetbuff ){
free ( offsetbuff );
}
if ( NULL != ind_ts ) {
free ( ind_ts );
}
if ( NULL != ind_recordlength ) {
free ( ind_recordlength );
}
if ( NULL != local_off ) {
free ( local_off );
}
if ( NULL != buff ) {
free ( buff );
}
return ret;
}
