int mca_common_ompio_set_file_defaults (mca_io_ompio_file_t *fh) { if (NULL != fh) { ompi_datatype_t *types[2]; int blocklen[2] = {1, 1}; OPAL_PTRDIFF_TYPE d[2], base; int i; fh->f_io_array = NULL; fh->f_perm = OMPIO_PERM_NULL; fh->f_flags = 0; fh->f_bytes_per_agg = mca_io_ompio_bytes_per_agg; fh->f_datarep = strdup ("native"); fh->f_offset = 0; fh->f_disp = 0; fh->f_position_in_file_view = 0; fh->f_index_in_file_view = 0; fh->f_total_bytes = 0; fh->f_init_procs_per_group = -1; fh->f_init_procs_in_group = NULL; fh->f_procs_per_group = -1; fh->f_procs_in_group = NULL; fh->f_init_num_aggrs = -1; fh->f_init_aggr_list = NULL; /* Default file View */ fh->f_iov_type = MPI_DATATYPE_NULL; fh->f_stripe_size = mca_io_ompio_bytes_per_agg; /*Decoded iovec of the file-view*/ fh->f_decoded_iov = NULL; fh->f_etype = NULL; fh->f_filetype = NULL; fh->f_orig_filetype = NULL; mca_common_ompio_set_view(fh, 0, &ompi_mpi_byte.dt, &ompi_mpi_byte.dt, "native", fh->f_info); /*Create a derived datatype for the created iovec */ types[0] = &ompi_mpi_long.dt; types[1] = &ompi_mpi_long.dt; d[0] = (OPAL_PTRDIFF_TYPE) fh->f_decoded_iov; d[1] = (OPAL_PTRDIFF_TYPE) &fh->f_decoded_iov[0].iov_len; base = d[0]; for (i=0 ; i<2 ; i++) { d[i] -= base; } ompi_datatype_create_struct (2, blocklen, d, types, &fh->f_iov_type); ompi_datatype_commit (&fh->f_iov_type); return OMPI_SUCCESS; } else { return OMPI_ERROR; } }
int mca_sharedfp_addproc_file_open (struct ompi_communicator_t *comm, const char* filename, int amode, struct ompi_info_t *info, mca_io_ompio_file_t *fh) { int ret = OMPI_SUCCESS, err; int rank; struct mca_sharedfp_base_data_t* sh; mca_io_ompio_file_t * shfileHandle, *ompio_fh; MPI_Comm newInterComm; struct mca_sharedfp_addproc_data * addproc_data = NULL; mca_io_ompio_data_t *data; /*-------------------------------------------------*/ /*Open the same file again without shared file pointer*/ /*-------------------------------------------------*/ shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); ret = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false); if ( OMPI_SUCCESS != ret) { opal_output(0, "mca_sharedfp_addproc_file_open: Error during file open\n"); return ret; } shfileHandle->f_fh = fh->f_fh; data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data; ompio_fh = &data->ompio_fh; err = mca_common_ompio_set_view (shfileHandle, ompio_fh->f_disp, ompio_fh->f_etype, ompio_fh->f_orig_filetype, ompio_fh->f_datarep, MPI_INFO_NULL); /*Memory is allocated here for the sh structure*/ if ( mca_sharedfp_addproc_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: malloc f_sharedfp_ptr struct\n"); } sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t)); if ( NULL == sh ){ opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: Error, unable to malloc f_sharedfp_ptr struct\n"); return OMPI_ERR_OUT_OF_RESOURCE; } /*Populate the sh file structure based on the implementation*/ sh->sharedfh = shfileHandle; /* Shared file pointer*/ sh->global_offset = 0; /* Global Offset*/ sh->comm = comm; /* Communicator*/ sh->selected_module_data = NULL; rank = ompi_comm_rank ( sh->comm ); if ( mca_sharedfp_addproc_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: START spawn by rank=%d\n",rank); } /*Spawn a new process which will maintain the offsets for this file open*/ ret = MPI_Comm_spawn("mca_sharedfp_addproc_control", MPI_ARGV_NULL, 1, MPI_INFO_NULL, 0, sh->comm, &newInterComm, &err); if ( OMPI_SUCCESS != ret ) { opal_output(0, "mca_sharedfp_addproc_file_open: error spawning control process ret=%d\n", ret); } /*If spawning successful*/ if (newInterComm) { addproc_data = (struct mca_sharedfp_addproc_data*)malloc(sizeof(struct mca_sharedfp_addproc_data)); if ( NULL == addproc_data ){ opal_output (0,"mca_sharedfp_addproc_file_open: Error, unable to malloc addproc_data struct\n"); return OMPI_ERR_OUT_OF_RESOURCE; } /*Store the new Intercommunicator*/ addproc_data->intercom = newInterComm; /*save the addproc data*/ sh->selected_module_data = addproc_data; /*remember the shared file handle*/ fh->f_sharedfp_data = sh; } else{ opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_addproc_file_open: DONE spawn by rank=%d, errcode[success=%d, err=%d]=%d\n", rank, MPI_SUCCESS, MPI_ERR_SPAWN, ret); ret = OMPI_ERROR; } return ret; }
int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm, const char* filename, int amode, struct opal_info_t *info, mca_io_ompio_file_t *fh) { int err = MPI_SUCCESS; char * lockedfilename; int handle, rank; struct mca_sharedfp_lockedfile_data * module_data = NULL; struct mca_sharedfp_base_data_t* sh; mca_io_ompio_file_t * shfileHandle, *ompio_fh; mca_io_ompio_data_t *data; /*------------------------------------------------------------*/ /*Open the same file again without shared file pointer support*/ /*------------------------------------------------------------*/ shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t)); err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false); if ( OMPI_SUCCESS != err) { opal_output(0, "mca_sharedfp_lockedfile_file_open: Error during file open\n"); return err; } shfileHandle->f_fh = fh->f_fh; data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data; ompio_fh = &data->ompio_fh; err = mca_common_ompio_set_view (shfileHandle, ompio_fh->f_disp, ompio_fh->f_etype, ompio_fh->f_orig_filetype, ompio_fh->f_datarep, &(MPI_INFO_NULL->super)); /*Memory is allocated here for the sh structure*/ sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t)); if ( NULL == sh){ opal_output(0, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp_ptr struct\n"); free ( shfileHandle); return OMPI_ERR_OUT_OF_RESOURCE; } /*Populate the sh file structure based on the implementation*/ sh->sharedfh = shfileHandle; /* Shared file pointer*/ sh->global_offset = 0; /* Global Offset*/ sh->comm = comm; /* Communicator*/ sh->selected_module_data = NULL; rank = ompi_comm_rank ( sh->comm); /*Open a new file which will maintain the pointer for this file open*/ if ( mca_sharedfp_lockedfile_verbose ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_lockedfile_file_open: open locked file.\n"); } module_data = (struct mca_sharedfp_lockedfile_data*)malloc(sizeof(struct mca_sharedfp_lockedfile_data)); if ( NULL == module_data ) { opal_output(ompi_sharedfp_base_framework.framework_output, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc lockedfile_data struct\n"); free (shfileHandle); free (sh); return OMPI_ERR_OUT_OF_RESOURCE; } opal_jobid_t masterjobid; if ( 0 == comm->c_my_rank ) { ompi_proc_t *masterproc = ompi_group_peer_lookup(comm->c_local_group, 0 ); masterjobid = OMPI_CAST_RTE_NAME(&masterproc->super.proc_name)->jobid; } comm->c_coll->coll_bcast ( &masterjobid, 1, MPI_UNSIGNED, 0, comm, comm->c_coll->coll_bcast_module ); size_t filenamelen = strlen(filename) + 16; lockedfilename = (char*)malloc(sizeof(char) * filenamelen); if ( NULL == lockedfilename ) { free (shfileHandle); free (sh); free (module_data); return OMPI_ERR_OUT_OF_RESOURCE; } snprintf(lockedfilename, filenamelen, "%s-%u%s",filename,masterjobid,".lock"); module_data->filename = lockedfilename; /*-------------------------------------------------*/ /*Open the lockedfile without shared file pointer */ /*-------------------------------------------------*/ if ( 0 == rank ) { OMPI_MPI_OFFSET_TYPE position=0; /*only let main process initialize file pointer, *therefore there is no need to lock the file */ handle = open ( lockedfilename, O_RDWR | O_CREAT, 0644 ); write ( handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE) ); close ( handle ); } comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module ); handle = open ( lockedfilename, O_RDWR, 0644 ); if ( -1 == handle ) { opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", rank); free (shfileHandle); free (sh); free(module_data); return OMPI_ERROR; } /*Store the new file handle*/ module_data->handle = handle; /* Assign the lockedfile_data to sh->handle*/ sh->selected_module_data = module_data; /*remember the shared file handle*/ fh->f_sharedfp_data = sh; comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module ); return err; }