int mca_common_ompio_set_file_defaults (mca_io_ompio_file_t *fh)
{
if (NULL != fh) {
ompi_datatype_t *types[2];
int blocklen[2] = {1, 1};
OPAL_PTRDIFF_TYPE d[2], base;
int i;
fh->f_io_array = NULL;
fh->f_perm = OMPIO_PERM_NULL;
fh->f_flags = 0;
fh->f_bytes_per_agg = mca_io_ompio_bytes_per_agg;
fh->f_datarep = strdup ("native");
fh->f_offset = 0;
fh->f_disp = 0;
fh->f_position_in_file_view = 0;
fh->f_index_in_file_view = 0;
fh->f_total_bytes = 0;
fh->f_init_procs_per_group = -1;
fh->f_init_procs_in_group = NULL;
fh->f_procs_per_group = -1;
fh->f_procs_in_group = NULL;
fh->f_init_num_aggrs = -1;
fh->f_init_aggr_list = NULL;
/* Default file View */
fh->f_iov_type = MPI_DATATYPE_NULL;
fh->f_stripe_size = mca_io_ompio_bytes_per_agg;
/*Decoded iovec of the file-view*/
fh->f_decoded_iov = NULL;
fh->f_etype = NULL;
fh->f_filetype = NULL;
fh->f_orig_filetype = NULL;
mca_common_ompio_set_view(fh,
0,
&ompi_mpi_byte.dt,
&ompi_mpi_byte.dt,
"native",
fh->f_info);
/*Create a derived datatype for the created iovec */
types[0] = &ompi_mpi_long.dt;
types[1] = &ompi_mpi_long.dt;
d[0] = (OPAL_PTRDIFF_TYPE) fh->f_decoded_iov;
d[1] = (OPAL_PTRDIFF_TYPE) &fh->f_decoded_iov[0].iov_len;
base = d[0];
for (i=0 ; i<2 ; i++) {
d[i] -= base;
}
ompi_datatype_create_struct (2,
blocklen,
d,
types,
&fh->f_iov_type);
ompi_datatype_commit (&fh->f_iov_type);
return OMPI_SUCCESS;
}
else {
return OMPI_ERROR;
}
}
int mca_sharedfp_addproc_file_open (struct ompi_communicator_t *comm,
const char* filename,
int amode,
struct ompi_info_t *info,
mca_io_ompio_file_t *fh)
{
int ret = OMPI_SUCCESS, err;
int rank;
struct mca_sharedfp_base_data_t* sh;
mca_io_ompio_file_t * shfileHandle, *ompio_fh;
MPI_Comm newInterComm;
struct mca_sharedfp_addproc_data * addproc_data = NULL;
mca_io_ompio_data_t *data;
/*-------------------------------------------------*/
/*Open the same file again without shared file pointer*/
/*-------------------------------------------------*/
shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
ret = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
if ( OMPI_SUCCESS != ret) {
opal_output(0, "mca_sharedfp_addproc_file_open: Error during file open\n");
return ret;
}
shfileHandle->f_fh = fh->f_fh;
data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data;
ompio_fh = &data->ompio_fh;
err = mca_common_ompio_set_view (shfileHandle,
ompio_fh->f_disp,
ompio_fh->f_etype,
ompio_fh->f_orig_filetype,
ompio_fh->f_datarep,
MPI_INFO_NULL);
/*Memory is allocated here for the sh structure*/
if ( mca_sharedfp_addproc_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: malloc f_sharedfp_ptr struct\n");
}
sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
if ( NULL == sh ){
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Populate the sh file structure based on the implementation*/
sh->sharedfh = shfileHandle; /* Shared file pointer*/
sh->global_offset = 0; /* Global Offset*/
sh->comm = comm; /* Communicator*/
sh->selected_module_data = NULL;
rank = ompi_comm_rank ( sh->comm );
if ( mca_sharedfp_addproc_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: START spawn by rank=%d\n",rank);
}
/*Spawn a new process which will maintain the offsets for this file open*/
ret = MPI_Comm_spawn("mca_sharedfp_addproc_control", MPI_ARGV_NULL, 1, MPI_INFO_NULL,
0, sh->comm, &newInterComm, &err);
if ( OMPI_SUCCESS != ret ) {
opal_output(0, "mca_sharedfp_addproc_file_open: error spawning control process ret=%d\n",
ret);
}
/*If spawning successful*/
if (newInterComm) {
addproc_data = (struct mca_sharedfp_addproc_data*)malloc(sizeof(struct mca_sharedfp_addproc_data));
if ( NULL == addproc_data ){
opal_output (0,"mca_sharedfp_addproc_file_open: Error, unable to malloc addproc_data struct\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Store the new Intercommunicator*/
addproc_data->intercom = newInterComm;
/*save the addproc data*/
sh->selected_module_data = addproc_data;
/*remember the shared file handle*/
fh->f_sharedfp_data = sh;
}
else{
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_addproc_file_open: DONE spawn by rank=%d, errcode[success=%d, err=%d]=%d\n",
rank, MPI_SUCCESS, MPI_ERR_SPAWN, ret);
ret = OMPI_ERROR;
}
return ret;
}
int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
const char* filename,
int amode,
struct opal_info_t *info,
mca_io_ompio_file_t *fh)
{
int err = MPI_SUCCESS;
char * lockedfilename;
int handle, rank;
struct mca_sharedfp_lockedfile_data * module_data = NULL;
struct mca_sharedfp_base_data_t* sh;
mca_io_ompio_file_t * shfileHandle, *ompio_fh;
mca_io_ompio_data_t *data;
/*------------------------------------------------------------*/
/*Open the same file again without shared file pointer support*/
/*------------------------------------------------------------*/
shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
if ( OMPI_SUCCESS != err) {
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error during file open\n");
return err;
}
shfileHandle->f_fh = fh->f_fh;
data = (mca_io_ompio_data_t *) fh->f_fh->f_io_selected_data;
ompio_fh = &data->ompio_fh;
err = mca_common_ompio_set_view (shfileHandle,
ompio_fh->f_disp,
ompio_fh->f_etype,
ompio_fh->f_orig_filetype,
ompio_fh->f_datarep,
&(MPI_INFO_NULL->super));
/*Memory is allocated here for the sh structure*/
sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
if ( NULL == sh){
opal_output(0, "mca_sharedfp_lockedfile_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
free ( shfileHandle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Populate the sh file structure based on the implementation*/
sh->sharedfh = shfileHandle; /* Shared file pointer*/
sh->global_offset = 0; /* Global Offset*/
sh->comm = comm; /* Communicator*/
sh->selected_module_data = NULL;
rank = ompi_comm_rank ( sh->comm);
/*Open a new file which will maintain the pointer for this file open*/
if ( mca_sharedfp_lockedfile_verbose ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_lockedfile_file_open: open locked file.\n");
}
module_data = (struct mca_sharedfp_lockedfile_data*)malloc(sizeof(struct mca_sharedfp_lockedfile_data));
if ( NULL == module_data ) {
opal_output(ompi_sharedfp_base_framework.framework_output,
"mca_sharedfp_lockedfile_file_open: Error, unable to malloc lockedfile_data struct\n");
free (shfileHandle);
free (sh);
return OMPI_ERR_OUT_OF_RESOURCE;
}
opal_jobid_t masterjobid;
if ( 0 == comm->c_my_rank ) {
ompi_proc_t *masterproc = ompi_group_peer_lookup(comm->c_local_group, 0 );
masterjobid = OMPI_CAST_RTE_NAME(&masterproc->super.proc_name)->jobid;
}
comm->c_coll->coll_bcast ( &masterjobid, 1, MPI_UNSIGNED, 0, comm,
comm->c_coll->coll_bcast_module );
size_t filenamelen = strlen(filename) + 16;
lockedfilename = (char*)malloc(sizeof(char) * filenamelen);
if ( NULL == lockedfilename ) {
free (shfileHandle);
free (sh);
free (module_data);
return OMPI_ERR_OUT_OF_RESOURCE;
}
snprintf(lockedfilename, filenamelen, "%s-%u%s",filename,masterjobid,".lock");
module_data->filename = lockedfilename;
/*-------------------------------------------------*/
/*Open the lockedfile without shared file pointer */
/*-------------------------------------------------*/
if ( 0 == rank ) {
OMPI_MPI_OFFSET_TYPE position=0;
/*only let main process initialize file pointer,
*therefore there is no need to lock the file
*/
handle = open ( lockedfilename, O_RDWR | O_CREAT, 0644 );
write ( handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE) );
close ( handle );
}
comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module );
handle = open ( lockedfilename, O_RDWR, 0644 );
if ( -1 == handle ) {
opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", rank);
free (shfileHandle);
free (sh);
free(module_data);
return OMPI_ERROR;
}
/*Store the new file handle*/
module_data->handle = handle;
/* Assign the lockedfile_data to sh->handle*/
sh->selected_module_data = module_data;
/*remember the shared file handle*/
fh->f_sharedfp_data = sh;
comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module );
return err;
}
