int mca_io_ompio_file_open (ompi_communicator_t *comm,
const char *filename,
int amode,
ompi_info_t *info,
ompi_file_t *fh)
{
int ret = OMPI_SUCCESS;
mca_io_ompio_data_t *data=NULL;
bool use_sharedfp = true;
data = (mca_io_ompio_data_t *) fh->f_io_selected_data;
if ( NULL == data ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*save pointer back to the file_t structure */
data->ompio_fh.f_fh = fh;
ret = mca_common_ompio_file_open(comm,filename,amode,info,&data->ompio_fh,use_sharedfp);
if ( OMPI_SUCCESS == ret ) {
fh->f_flags |= OMPIO_FILE_IS_OPEN;
}
return ret;
}
int mca_io_ompio_file_open (ompi_communicator_t *comm,
const char *filename,
int amode,
opal_info_t *info,
ompi_file_t *fh)
{
int ret = OMPI_SUCCESS;
mca_io_ompio_data_t *data=NULL;
bool use_sharedfp = true;
/* No locking required for file_open according to my understanding
There is virtually no way on how to reach this point from multiple
threads simultaniously
*/
data = (mca_io_ompio_data_t *) fh->f_io_selected_data;
if ( NULL == data ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*save pointer back to the file_t structure */
data->ompio_fh.f_fh = fh;
/* No lock required for file_open even in multi-threaded scenarios,
since only one collective operation per communicator
is allowed anyway */
ret = mca_common_ompio_file_open(comm,filename,amode,info,&data->ompio_fh,use_sharedfp);
if ( OMPI_SUCCESS == ret ) {
fh->f_flags |= OMPIO_FILE_IS_OPEN;
}
return ret;
}
int mca_sharedfp_addproc_file_open (struct ompi_communicator_t *comm,
const char* filename,
int amode,
struct ompi_info_t *info,
mca_io_ompio_file_t *fh)
{
int ret = OMPI_SUCCESS, err;
int rank;
struct mca_sharedfp_base_data_t* sh;
mca_io_ompio_file_t * shfileHandle, *ompio_fh;
MPI_Comm newInterComm;
struct mca_sharedfp_addproc_data * addproc_data = NULL;
mca_io_ompio_data_t *data;
/*-------------------------------------------------*/
/*Open the same file again without shared file pointer*/
/*-------------------------------------------------*/
shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
ret = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
if ( OMPI_SUCCESS != ret) {
opal_output(0, "mca_sharedfp_addproc_file_open: Error during file open\n");
return ret;
}
shfileHandle->f_fh = fh->f_fh;
data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data;
ompio_fh = &data->ompio_fh;
err = mca_common_ompio_set_view (shfileHandle,
ompio_fh->f_disp,
ompio_fh->f_etype,
ompio_fh->f_orig_filetype,
ompio_fh->f_datarep,
MPI_INFO_NULL);
/*Memory is allocated here for the sh structure*/
if ( mca_sharedfp_addproc_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: malloc f_sharedfp_ptr struct\n");
}
sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
if ( NULL == sh ){
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Populate the sh file structure based on the implementation*/
sh->sharedfh = shfileHandle; /* Shared file pointer*/
sh->global_offset = 0; /* Global Offset*/
sh->comm = comm; /* Communicator*/
sh->selected_module_data = NULL;
rank = ompi_comm_rank ( sh->comm );
if ( mca_sharedfp_addproc_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: START spawn by rank=%d\n",rank);
}
/*Spawn a new process which will maintain the offsets for this file open*/
ret = MPI_Comm_spawn("mca_sharedfp_addproc_control", MPI_ARGV_NULL, 1, MPI_INFO_NULL,
0, sh->comm, &newInterComm, &err);
if ( OMPI_SUCCESS != ret ) {
opal_output(0, "mca_sharedfp_addproc_file_open: error spawning control process ret=%d\n",
ret);
}
/*If spawning successful*/
if (newInterComm) {
addproc_data = (struct mca_sharedfp_addproc_data*)malloc(sizeof(struct mca_sharedfp_addproc_data));
if ( NULL == addproc_data ){
opal_output (0,"mca_sharedfp_addproc_file_open: Error, unable to malloc addproc_data struct\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Store the new Intercommunicator*/
addproc_data->intercom = newInterComm;
/*save the addproc data*/
sh->selected_module_data = addproc_data;
/*remember the shared file handle*/
fh->f_sharedfp_data = sh;
}
else{
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: DONE spawn by rank=%d, errcode[success=%d, err=%d]=%d\n",
rank, MPI_SUCCESS, MPI_ERR_SPAWN, ret);
ret = OMPI_ERROR;
}
return ret;
}
int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm,
const char* filename,
int amode,
struct ompi_info_t *info,
mca_io_ompio_file_t *fh)
{
int err = 0;
int rank;
char * datafilename; /*The array size would change as it is based on the current path*/
char * metadatafilename; /*The array size would change as it is based on the current path*/
mca_io_ompio_file_t * datafilehandle;
mca_io_ompio_file_t * metadatafilehandle;
mca_sharedfp_individual_header_record* headnode = NULL;
struct mca_sharedfp_base_data_t* sh;
mca_io_ompio_file_t * shfileHandle;
size_t len=0;
/*-------------------------------------------------*/
/*Open the same file again without shared file pointer*/
/*-------------------------------------------------*/
shfileHandle = (mca_io_ompio_file_t *) malloc ( sizeof(mca_io_ompio_file_t));
if ( NULL == shfileHandle ) {
opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
err = mca_common_ompio_file_open ( comm, filename, amode, info, shfileHandle, false);
if ( OMPI_SUCCESS != err ) {
opal_output(0, "mca_sharedfp_individual_file_open: Error during file open\n");
return err;
}
shfileHandle->f_fh = fh->f_fh;
sh = (struct mca_sharedfp_base_data_t*) malloc ( sizeof(struct mca_sharedfp_base_data_t));
if ( NULL == sh ){
opal_output(0, "mca_sharedfp_individual_file_open: Error, unable to malloc "
"f_sharedfp_ptr struct\n");
free ( shfileHandle );
return OMPI_ERR_OUT_OF_RESOURCE;
}
rank = ompi_comm_rank ( comm );
/*Populate the sh file structure based on the implementation*/
sh->sharedfh = shfileHandle; /* Shared file pointer*/
sh->global_offset = 0; /* Global Offset*/
sh->comm = comm; /* Communicator*/
sh->selected_module_data = NULL;
/* Assign the metadatalinked list to sh->handle */
sh->selected_module_data = (mca_sharedfp_individual_header_record *)mca_sharedfp_individual_insert_headnode();
/*--------------------------------------------------------*/
/* Open an individual data file for each process */
/* NOTE: Open the data file without shared file pointer */
/*--------------------------------------------------------*/
if ( mca_sharedfp_individual_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_individual_file_open: open data file.\n");
}
/* data filename created by appending .data.$rank to the original filename*/
len = strlen (filename ) + 64;
datafilename = (char*)malloc( len );
if ( NULL == datafilename ) {
opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n");
free ( shfileHandle );
free ( sh );
return OMPI_ERR_OUT_OF_RESOURCE;
}
snprintf(datafilename, len, "%s%s%d",filename,".data.",rank);
datafilehandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
if ( NULL == datafilehandle ) {
opal_output(0, "mca_sharedfp_individual_file_open: unable to allocate memory\n");
free ( shfileHandle );
free ( sh );
free ( datafilename );
return OMPI_ERR_OUT_OF_RESOURCE;
}
err = mca_common_ompio_file_open(MPI_COMM_SELF, datafilename,
MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE,
MPI_INFO_NULL, datafilehandle, false);
if ( OMPI_SUCCESS != err) {
opal_output(0, "mca_sharedfp_individual_file_open: Error during datafile file open\n");
free (shfileHandle );
free (sh);
free (datafilename);
free (datafilehandle);
return err;
}
/*----------------------------------------------------------*/
/* Open an individual metadata file for each of the process */
/* NOTE: Open the meta file without shared file pointer */
/*----------------------------------------------------------*/
if ( mca_sharedfp_individual_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_individual_file_open: metadata file.\n");
}
/* metadata filename created by appending .metadata.$rank to the original filename*/
metadatafilename = (char*) malloc ( len );
if ( NULL == metadatafilename ) {
free (shfileHandle );
free (sh);
free (datafilename);
free (datafilehandle);
opal_output(0, "mca_sharedfp_individual_file_open: Error during memory allocation\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
snprintf ( metadatafilename, len, "%s%s%d", filename, ".metadata.",rank);
metadatafilehandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
if ( NULL == metadatafilehandle ) {
free (shfileHandle );
free (sh);
free (datafilename);
free (datafilehandle);
free (metadatafilename);
opal_output(0, "mca_sharedfp_individual_file_open: Error during memory allocation\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
err = mca_common_ompio_file_open ( MPI_COMM_SELF,metadatafilename,
MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE,
MPI_INFO_NULL, metadatafilehandle, false);
if ( OMPI_SUCCESS != err) {
opal_output(0, "mca_sharedfp_individual_file_open: Error during metadatafile file open\n");
free (shfileHandle );
free (sh);
free (datafilename);
free (datafilehandle);
free (metadatafilename);
free (metadatafilehandle);
return err;
}
/*save the datafilehandle and metadatahandle in the sharedfp individual module data structure*/
headnode = (mca_sharedfp_individual_header_record*)(sh->selected_module_data);
if ( headnode) {
headnode->datafilehandle = datafilehandle;
headnode->metadatafilehandle = metadatafilehandle;
headnode->datafilename = datafilename;
headnode->metadatafilename = metadatafilename;
}
/*save the sharedfp individual module data structure in the ompio filehandle structure*/
fh->f_sharedfp_data = sh;
return err;
}
int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
const char* filename,
int amode,
struct opal_info_t *info,
mca_io_ompio_file_t *fh)
{
int err = MPI_SUCCESS;
char * lockedfilename;
int handle, rank;
struct mca_sharedfp_lockedfile_data * module_data = NULL;
struct mca_sharedfp_base_data_t* sh;
mca_io_ompio_file_t * shfileHandle, *ompio_fh;
mca_io_ompio_data_t *data;
/*------------------------------------------------------------*/
/*Open the same file again without shared file pointer support*/
/*------------------------------------------------------------*/
shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
if ( OMPI_SUCCESS != err) {
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error during file open\n");
return err;
}
shfileHandle->f_fh = fh->f_fh;
data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data;
ompio_fh = &data->ompio_fh;
err = mca_common_ompio_set_view (shfileHandle,
ompio_fh->f_disp,
ompio_fh->f_etype,
ompio_fh->f_orig_filetype,
ompio_fh->f_datarep,
&(MPI_INFO_NULL->super));
/*Memory is allocated here for the sh structure*/
sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
if ( NULL == sh){
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
free ( shfileHandle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Populate the sh file structure based on the implementation*/
sh->sharedfh = shfileHandle; /* Shared file pointer*/
sh->global_offset = 0; /* Global Offset*/
sh->comm = comm; /* Communicator*/
sh->selected_module_data = NULL;
rank = ompi_comm_rank ( sh->comm);
/*Open a new file which will maintain the pointer for this file open*/
if ( mca_sharedfp_lockedfile_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_lockedfile_file_open: open locked file.\n");
}
module_data = (struct mca_sharedfp_lockedfile_data*)malloc(sizeof(struct mca_sharedfp_lockedfile_data));
if ( NULL == module_data ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_lockedfile_file_open: Error, unable to malloc lockedfile_data struct\n");
free (shfileHandle);
free (sh);
return OMPI_ERR_OUT_OF_RESOURCE;
}
opal_jobid_t masterjobid;
if ( 0 == comm->c_my_rank ) {
ompi_proc_t *masterproc = ompi_group_peer_lookup(comm->c_local_group, 0 );
masterjobid = OMPI_CAST_RTE_NAME(&masterproc->super.proc_name)->jobid;
}
comm->c_coll->coll_bcast ( &masterjobid, 1, MPI_UNSIGNED, 0, comm,
comm->c_coll->coll_bcast_module );
size_t filenamelen = strlen(filename) + 16;
lockedfilename = (char*)malloc(sizeof(char) * filenamelen);
if ( NULL == lockedfilename ) {
free (shfileHandle);
free (sh);
free (module_data);
return OMPI_ERR_OUT_OF_RESOURCE;
}
snprintf(lockedfilename, filenamelen, "%s-%u%s",filename,masterjobid,".lock");
module_data->filename = lockedfilename;
/*-------------------------------------------------*/
/*Open the lockedfile without shared file pointer */
/*-------------------------------------------------*/
if ( 0 == rank ) {
OMPI_MPI_OFFSET_TYPE position=0;
/*only let main process initialize file pointer,
*therefore there is no need to lock the file
*/
handle = open ( lockedfilename, O_RDWR | O_CREAT, 0644 );
write ( handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE) );
close ( handle );
}
comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module );
handle = open ( lockedfilename, O_RDWR, 0644 );
if ( -1 == handle ) {
opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", rank);
free (shfileHandle);
free (sh);
free(module_data);
return OMPI_ERROR;
}
/*Store the new file handle*/
module_data->handle = handle;
/* Assign the lockedfile_data to sh->handle*/
sh->selected_module_data = module_data;
/*remember the shared file handle*/
fh->f_sharedfp_data = sh;
comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module );
return err;
}
