int mca_io_ompio_file_open (ompi_communicator_t *comm, const char *filename, int amode, ompi_info_t *info, ompi_file_t *fh) { int ret = OMPI_SUCCESS; mca_io_ompio_data_t *data=NULL; bool use_sharedfp = true; data = (mca_io_ompio_data_t *) fh->f_io_selected_data; if ( NULL == data ) { return OMPI_ERR_OUT_OF_RESOURCE; } /*save pointer back to the file_t structure */ data->ompio_fh.f_fh = fh; ret = mca_common_ompio_file_open(comm,filename,amode,info,&data->ompio_fh,use_sharedfp); if ( OMPI_SUCCESS == ret ) { fh->f_flags |= OMPIO_FILE_IS_OPEN; } return ret; }
int mca_io_ompio_file_open (ompi_communicator_t *comm, const char *filename, int amode, opal_info_t *info, ompi_file_t *fh) { int ret = OMPI_SUCCESS; mca_io_ompio_data_t *data=NULL; bool use_sharedfp = true; /* No locking required for file_open according to my understanding There is virtually no way on how to reach this point from multiple threads simultaniously */ data = (mca_io_ompio_data_t *) fh->f_io_selected_data; if ( NULL == data ) { return OMPI_ERR_OUT_OF_RESOURCE; } /*save pointer back to the file_t structure */ data->ompio_fh.f_fh = fh; /* No lock required for file_open even in multi-threaded scenarios, since only one collective operation per communicator is allowed anyway */ ret = mca_common_ompio_file_open(comm,filename,amode,info,&data->ompio_fh,use_sharedfp); if ( OMPI_SUCCESS == ret ) { fh->f_flags |= OMPIO_FILE_IS_OPEN; } return ret; }
int mca_sharedfp_addproc_file_open (struct ompi_communicator_t *comm, const char* filename, int amode, struct ompi_info_t *info, mca_io_ompio_file_t *fh) { int ret = OMPI_SUCCESS, err; int rank; struct mca_sharedfp_base_data_t* sh; mca_io_ompio_file_t * shfileHandle, *ompio_fh; MPI_Comm newInterComm; struct mca_sharedfp_addproc_data * addproc_data = NULL; mca_io_ompio_data_t *data; /*-------------------------------------------------*/ /*Open the same file again without shared file pointer*/ /*-------------------------------------------------*/ shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); ret = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false); if ( OMPI_SUCCESS != ret) { opal_output(0, "mca_sharedfp_addproc_file_open: Error during file open\n"); return ret; } shfileHandle->f_fh = fh->f_fh; data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data; ompio_fh = &data->ompio_fh; err = mca_common_ompio_set_view (shfileHandle, ompio_fh->f_disp, ompio_fh->f_etype, ompio_fh->f_orig_filetype, ompio_fh->f_datarep, MPI_INFO_NULL); /*Memory is allocated here for the sh structure*/ if ( mca_sharedfp_addproc_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: malloc f_sharedfp_ptr struct\n"); } sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t)); if ( NULL == sh ){ opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: Error, unable to malloc f_sharedfp_ptr struct\n"); return OMPI_ERR_OUT_OF_RESOURCE; } /*Populate the sh file structure based on the implementation*/ sh->sharedfh = shfileHandle; /* Shared file pointer*/ sh->global_offset = 0; /* Global Offset*/ sh->comm = comm; /* Communicator*/ sh->selected_module_data = NULL; rank = ompi_comm_rank ( sh->comm ); if ( mca_sharedfp_addproc_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: START spawn by rank=%d\n",rank); } /*Spawn a new process which will maintain the offsets for this file open*/ ret = MPI_Comm_spawn("mca_sharedfp_addproc_control", MPI_ARGV_NULL, 1, MPI_INFO_NULL, 0, sh->comm, &newInterComm, &err); if ( OMPI_SUCCESS != ret ) { opal_output(0, "mca_sharedfp_addproc_file_open: error spawning control process ret=%d\n", ret); } /*If spawning successful*/ if (newInterComm) { addproc_data = (struct mca_sharedfp_addproc_data*)malloc(sizeof(struct mca_sharedfp_addproc_data)); if ( NULL == addproc_data ){ opal_output (0,"mca_sharedfp_addproc_file_open: Error, unable to malloc addproc_data struct\n"); return OMPI_ERR_OUT_OF_RESOURCE; } /*Store the new Intercommunicator*/ addproc_data->intercom = newInterComm; /*save the addproc data*/ sh->selected_module_data = addproc_data; /*remember the shared file handle*/ fh->f_sharedfp_data = sh; } else{ opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: DONE spawn by rank=%d, errcode[success=%d, err=%d]=%d\n", rank, MPI_SUCCESS, MPI_ERR_SPAWN, ret); ret = OMPI_ERROR; } return ret; }
int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm, const char* filename, int amode, struct ompi_info_t *info, mca_io_ompio_file_t *fh) { int err = 0; int rank; char * datafilename; /*The array size would change as it is based on the current path*/ char * metadatafilename; /*The array size would change as it is based on the current path*/ mca_io_ompio_file_t * datafilehandle; mca_io_ompio_file_t * metadatafilehandle; mca_sharedfp_individual_header_record* headnode = NULL; struct mca_sharedfp_base_data_t* sh; mca_io_ompio_file_t * shfileHandle; size_t len=0; /*-------------------------------------------------*/ /*Open the same file again without shared file pointer*/ /*-------------------------------------------------*/ shfileHandle = (mca_io_ompio_file_t *) malloc ( sizeof(mca_io_ompio_file_t)); if ( NULL == shfileHandle ) { opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n"); return OMPI_ERR_OUT_OF_RESOURCE; } err = mca_common_ompio_file_open ( comm, filename, amode, info, shfileHandle, false); if ( OMPI_SUCCESS != err ) { opal_output(0, "mca_sharedfp_individual_file_open: Error during file open\n"); return err; } shfileHandle->f_fh = fh->f_fh; sh = (struct mca_sharedfp_base_data_t*) malloc ( sizeof(struct mca_sharedfp_base_data_t)); if ( NULL == sh ){ opal_output(0, "mca_sharedfp_individual_file_open: Error, unable to malloc " "f_sharedfp_ptr struct\n"); free ( shfileHandle ); return OMPI_ERR_OUT_OF_RESOURCE; } rank = ompi_comm_rank ( comm ); /*Populate the sh file structure based on the implementation*/ sh->sharedfh = shfileHandle; /* Shared file pointer*/ sh->global_offset = 0; /* Global Offset*/ sh->comm = comm; /* Communicator*/ sh->selected_module_data = NULL; /* Assign the metadatalinked list to sh->handle */ sh->selected_module_data = (mca_sharedfp_individual_header_record *)mca_sharedfp_individual_insert_headnode(); /*--------------------------------------------------------*/ /* Open an individual data file for each process */ /* NOTE: Open the data file without shared file pointer */ /*--------------------------------------------------------*/ if ( mca_sharedfp_individual_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_individual_file_open: open data file.\n"); } /* data filename created by appending .data.$rank to the original filename*/ len = strlen (filename ) + 64; datafilename = (char*)malloc( len ); if ( NULL == datafilename ) { opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n"); free ( shfileHandle ); free ( sh ); return OMPI_ERR_OUT_OF_RESOURCE; } snprintf(datafilename, len, "%s%s%d",filename,".data.",rank); datafilehandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); if ( NULL == datafilehandle ) { opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n"); free ( shfileHandle ); free ( sh ); free ( datafilename ); return OMPI_ERR_OUT_OF_RESOURCE; } err = mca_common_ompio_file_open(MPI_COMM_SELF, datafilename, MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE, MPI_INFO_NULL, datafilehandle, false); if ( OMPI_SUCCESS != err) { opal_output(0, "mca_sharedfp_individual_file_open: Error during datafile file open\n"); free (shfileHandle ); free (sh); free (datafilename); free (datafilehandle); return err; } /*----------------------------------------------------------*/ /* Open an individual metadata file for each of the process */ /* NOTE: Open the meta file without shared file pointer */ /*----------------------------------------------------------*/ if ( mca_sharedfp_individual_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_individual_file_open: metadata file.\n"); } /* metadata filename created by appending .metadata.$rank to the original filename*/ metadatafilename = (char*) malloc ( len ); if ( NULL == metadatafilename ) { free (shfileHandle ); free (sh); free (datafilename); free (datafilehandle); opal_output(0, "mca_sharedfp_individual_file_open: Error during memory allocation\n"); return OMPI_ERR_OUT_OF_RESOURCE; } snprintf ( metadatafilename, len, "%s%s%d", filename, ".metadata.",rank); metadatafilehandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); if ( NULL == metadatafilehandle ) { free (shfileHandle ); free (sh); free (datafilename); free (datafilehandle); free (metadatafilename); opal_output(0, "mca_sharedfp_individual_file_open: Error during memory allocation\n"); return OMPI_ERR_OUT_OF_RESOURCE; } err = mca_common_ompio_file_open ( MPI_COMM_SELF,metadatafilename, MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE, MPI_INFO_NULL, metadatafilehandle, false); if ( OMPI_SUCCESS != err) { opal_output(0, "mca_sharedfp_individual_file_open: Error during metadatafile file open\n"); free (shfileHandle ); free (sh); free (datafilename); free (datafilehandle); free (metadatafilename); free (metadatafilehandle); return err; } /*save the datafilehandle and metadatahandle in the sharedfp individual module data structure*/ headnode = (mca_sharedfp_individual_header_record*)(sh->selected_module_data); if ( headnode) { headnode->datafilehandle = datafilehandle; headnode->metadatafilehandle = metadatafilehandle; headnode->datafilename = datafilename; headnode->metadatafilename = metadatafilename; } /*save the sharedfp individual module data structure in the ompio filehandle structure*/ fh->f_sharedfp_data = sh; return err; }
int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm, const char* filename, int amode, struct opal_info_t *info, mca_io_ompio_file_t *fh) { int err = MPI_SUCCESS; char * lockedfilename; int handle, rank; struct mca_sharedfp_lockedfile_data * module_data = NULL; struct mca_sharedfp_base_data_t* sh; mca_io_ompio_file_t * shfileHandle, *ompio_fh; mca_io_ompio_data_t *data; /*------------------------------------------------------------*/ /*Open the same file again without shared file pointer support*/ /*------------------------------------------------------------*/ shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false); if ( OMPI_SUCCESS != err) { opal_output(0, "mca_sharedfp_lockedfile_file_open: Error during file open\n"); return err; } shfileHandle->f_fh = fh->f_fh; data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data; ompio_fh = &data->ompio_fh; err = mca_common_ompio_set_view (shfileHandle, ompio_fh->f_disp, ompio_fh->f_etype, ompio_fh->f_orig_filetype, ompio_fh->f_datarep, &(MPI_INFO_NULL->super)); /*Memory is allocated here for the sh structure*/ sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t)); if ( NULL == sh){ opal_output(0, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp_ptr struct\n"); free ( shfileHandle); return OMPI_ERR_OUT_OF_RESOURCE; } /*Populate the sh file structure based on the implementation*/ sh->sharedfh = shfileHandle; /* Shared file pointer*/ sh->global_offset = 0; /* Global Offset*/ sh->comm = comm; /* Communicator*/ sh->selected_module_data = NULL; rank = ompi_comm_rank ( sh->comm); /*Open a new file which will maintain the pointer for this file open*/ if ( mca_sharedfp_lockedfile_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_lockedfile_file_open: open locked file.\n"); } module_data = (struct mca_sharedfp_lockedfile_data*)malloc(sizeof(struct mca_sharedfp_lockedfile_data)); if ( NULL == module_data ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc lockedfile_data struct\n"); free (shfileHandle); free (sh); return OMPI_ERR_OUT_OF_RESOURCE; } opal_jobid_t masterjobid; if ( 0 == comm->c_my_rank ) { ompi_proc_t *masterproc = ompi_group_peer_lookup(comm->c_local_group, 0 ); masterjobid = OMPI_CAST_RTE_NAME(&masterproc->super.proc_name)->jobid; } comm->c_coll->coll_bcast ( &masterjobid, 1, MPI_UNSIGNED, 0, comm, comm->c_coll->coll_bcast_module ); size_t filenamelen = strlen(filename) + 16; lockedfilename = (char*)malloc(sizeof(char) * filenamelen); if ( NULL == lockedfilename ) { free (shfileHandle); free (sh); free (module_data); return OMPI_ERR_OUT_OF_RESOURCE; } snprintf(lockedfilename, filenamelen, "%s-%u%s",filename,masterjobid,".lock"); module_data->filename = lockedfilename; /*-------------------------------------------------*/ /*Open the lockedfile without shared file pointer */ /*-------------------------------------------------*/ if ( 0 == rank ) { OMPI_MPI_OFFSET_TYPE position=0; /*only let main process initialize file pointer, *therefore there is no need to lock the file */ handle = open ( lockedfilename, O_RDWR | O_CREAT, 0644 ); write ( handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE) ); close ( handle ); } comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module ); handle = open ( lockedfilename, O_RDWR, 0644 ); if ( -1 == handle ) { opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", rank); free (shfileHandle); free (sh); free(module_data); return OMPI_ERROR; } /*Store the new file handle*/ module_data->handle = handle; /* Assign the lockedfile_data to sh->handle*/ sh->selected_module_data = module_data; /*remember the shared file handle*/ fh->f_sharedfp_data = sh; comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module ); return err; }