int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, const int dims[],
const long int start[], const long int count[], int *ioidp)
{
iosystem_desc_t *ios;
io_desc_t *iodesc;
int mpierr;
int ierr;
int iosize;
int ndisp;
for(int i=0;i<ndims;i++){
if(dims[i]<=0){
piodie("Invalid dims argument",__FILE__,__LINE__);
}
if(start[i]<0 || count[i]< 0 || (start[i]+count[i])>dims[i]){
piodie("Invalid start or count argument ",__FILE__,__LINE__);
}
}
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
int n, i, maplen=1;
for( i=0;i<ndims;i++){
maplen*=count[i];
}
PIO_Offset compmap[maplen], prod[ndims], loc[ndims];
prod[ndims-1]=1;
loc[ndims-1]=0;
for(n=ndims-2;n>=0;n--){
prod[n]=prod[n+1]*dims[n+1];
loc[n]=0;
}
for(i=0;i<maplen;i++){
compmap[i]=0;
for(n=ndims-1;n>=0;n--){
compmap[i]+=(start[n]+loc[n])*prod[n];
}
n=ndims-1;
loc[n]=(loc[n]+1)%count[n];
while(loc[n]==0 && n>0){
n--;
loc[n]=(loc[n]+1)%count[n];
}
}
int rearr = PIO_REARR_SUBSET;
PIOc_InitDecomp( iosysid, basetype,ndims, dims,
maplen, compmap, ioidp, &rearr, NULL, NULL);
return PIO_NOERR;
}
/**
** @brief return a logical indicating whether this task is an iotask
*/
int PIOc_iam_iotask(const int iosysid, bool *ioproc)
{
iosystem_desc_t *ios;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
*ioproc = ios->ioproc;
return PIO_NOERR;
}
/**
* @ingroup PIO_def_var
*
* Set chunk cache netCDF files to be opened/created.
*
* This function only applies to netCDF-4 files. When used with netCDF
* classic files, the error PIO_ENOTNC4 will be returned.
*
* The file chunk cache for HDF5 can be set, and will apply for any
* files opened or created until the program ends, or the settings are
* changed again. The cache settings apply only to the open file. They
* do not persist with the file, and must be set each time the file is
* opened, before it is opened, if they are to have effect.
*
* See the <a
* href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
* variable documentation</a> for details about the operation of this
* function.
*
* @param iotype the iotype of files to be created or opened.
* @param size size of file cache.
* @param nelems number of elements in file cache.
* @param preemption preemption setting for file cache.
*
* @return PIO_NOERR for success, otherwise an error code.
*/
int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size,
PIO_Offset nelems, float preemption)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
char *errstr;
errstr = NULL;
ierr = PIO_NOERR;
msg = PIO_MSG_SET_CHUNK_CACHE;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
/* if (ios->async_interface && ! ios->ioproc){ */
/* if (ios->compmaster) */
/* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */
/* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */
/* } */
switch (iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_set_chunk_cache(size, nelems, preemption);
break;
case PIO_IOTYPE_NETCDF4C:
if (!ios->io_rank)
ierr = nc_set_chunk_cache(size, nelems, preemption);
break;
#endif
case PIO_IOTYPE_NETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
/* Propogate error code to all processes. */
MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);
return ierr;
}
/**
** @brief Get the number of IO tasks set.
*/
int PIOc_get_numiotasks(int iosysid, int *numiotasks)
{
iosystem_desc_t *ios;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
*numiotasks = ios->num_iotasks;
return PIO_NOERR;
}
/**
** @brief return the rank of this task in the io comm or
** -1 if this task is not in the comm
*/
int PIOc_iotask_rank(const int iosysid, int *iorank)
{
iosystem_desc_t *ios;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
*iorank = ios->io_rank;
return PIO_NOERR;
}
/**
** @brief Send a hint to the MPI-IO library
**
*/
int PIOc_set_hint(const int iosysid, char hint[], const char hintval[])
{
iosystem_desc_t *ios;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
if(ios->ioproc)
CheckMPIReturn( MPI_Info_set(ios->info, hint, hintval), __FILE__,__LINE__);
return PIO_NOERR;
}
int PIOc_finalize(const int iosysid)
{
iosystem_desc_t *ios, *nios;
int msg;
int mpierr;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
/* If asynch IO is in use, send the PIO_MSG_EXIT message from the
* comp master to the IO processes. */
if (ios->async_interface && !ios->comp_rank)
{
msg = PIO_MSG_EXIT;
mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
CheckMPIReturn(mpierr, __FILE__, __LINE__);
}
/* Free this memory that was allocated in init_intracomm. */
if (ios->ioranks)
free(ios->ioranks);
/* Free the buffer pool. */
free_cn_buffer_pool(*ios);
/* Free the MPI groups. */
if (ios->compgroup != MPI_GROUP_NULL)
MPI_Group_free(&ios->compgroup);
if (ios->iogroup != MPI_GROUP_NULL)
MPI_Group_free(&(ios->iogroup));
/* Free the MPI communicators. my_comm is just a copy (but not an
* MPI copy), so does not have to have an MPI_Comm_free() call. */
if(ios->intercomm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->intercomm));
}
if(ios->io_comm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->io_comm));
}
if(ios->comp_comm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->comp_comm));
}
if(ios->union_comm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->union_comm));
}
/* Delete the iosystem_desc_t data associated with this id. */
return pio_delete_iosystem_from_list(iosysid);
}
/**
** @brief Check to see if PIO has been initialized.
*/
int PIOc_iosystem_is_active(const int iosysid, bool *active)
{
iosystem_desc_t *ios;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
if(ios->comp_comm == MPI_COMM_NULL && ios->io_comm == MPI_COMM_NULL){
*active = false;
}else{
*active = true;
}
return PIO_NOERR;
}
int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method)
{
iosystem_desc_t *ios;
int oldmethod;
ios = pio_get_iosystem_from_id(iosysid);
if(ios==NULL){
fprintf(stderr,"%s %d Error setting eh method\n",__FILE__,__LINE__);
print_trace(stderr);
return PIO_EBADID;
}
oldmethod = ios->error_handler;
ios->error_handler = method;
return(oldmethod);
}
/**
** @ingroup PIO_deletefile
** @brief Delete a file
** @param iosysid : a pio system handle
** @param filename : a filename
*/
int PIOc_deletefile(const int iosysid, const char filename[])
{
int ierr;
int msg;
int mpierr;
int chkerr;
iosystem_desc_t *ios;
ierr = PIO_NOERR;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
msg = 0;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
// mpierr = MPI_Bcast(iosysid,1, MPI_INT, ios->compmaster, ios->intercomm);
}
// The barriers are needed to assure that no task is trying to operate on the file while it is being deleted.
if(ios->ioproc){
MPI_Barrier(ios->io_comm);
#ifdef _NETCDF
if(ios->io_rank==0)
ierr = nc_delete(filename);
#else
#ifdef _PNETCDF
ierr = ncmpi_delete(filename, ios->info);
#endif
#endif
MPI_Barrier(ios->io_comm);
}
// Special case - always broadcast the return from the
MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm);
return ierr;
}
int PIOc_finalize(const int iosysid)
{
iosystem_desc_t *ios, *nios;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
/* FIXME: The memory for ioranks is allocated in C only for intracomms
* Remove this check once mem allocs for ioranks completely moves to the
* C code
*/
if(ios->intercomm == MPI_COMM_NULL){
if(ios->ioranks != NULL){
free(ios->ioranks);
}
}
free_cn_buffer_pool(*ios);
/* Free the MPI groups. */
MPI_Group_free(&(ios->compgroup));
MPI_Group_free(&(ios->iogroup));
/* Free the MPI communicators. */
if(ios->intercomm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->intercomm));
}
if(ios->io_comm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->io_comm));
}
if(ios->comp_comm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->comp_comm));
}
if(ios->union_comm != MPI_COMM_NULL){
MPI_Comm_free(&(ios->union_comm));
}
return pio_delete_iosystem_from_list(iosysid);
}
/**
* Test the decomp read/write functionality.
*
* @param iosysid the IO system ID.
* @param ioid the ID of the decomposition.
* @param num_flavors the number of IOTYPES available in this build.
* @param flavor array of available iotypes.
* @param my_rank rank of this task.
* @param rearranger the rearranger to use (PIO_REARR_BOX or
* PIO_REARR_SUBSET).
* @param test_comm the MPI communicator for this test.
* @returns 0 for success, error code otherwise.
*/
int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
int rearranger, MPI_Comm test_comm)
{
char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
int ioid2; /* ID for decomposition we will create from file. */
char title_in[PIO_MAX_NAME + 1]; /* Optional title. */
char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
int fortran_order_in; /* Indicates fortran vs. c order. */
int ret; /* Return code. */
/* Use PIO to create the decomp file in each of the four
* available ways. */
for (int fmt = 0; fmt < num_flavors; fmt++)
{
/* Create the filename. */
sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0)))
return ret;
/* Read the data. */
if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT,
title_in, history_in, &fortran_order_in)))
return ret;
/* Check the results. */
{
iosystem_desc_t *ios;
io_desc_t *iodesc;
/* Get the IO system info. */
if (!(ios = pio_get_iosystem_from_id(iosysid)))
return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
/* Get the IO desc, which describes the decomposition. */
if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
if (iodesc->ioid != ioid2 || iodesc->maplen != EXPECTED_MAPLEN || iodesc->ndims != NDIM3 ||
iodesc->ndof != EXPECTED_MAPLEN)
return ERR_WRONG;
if (iodesc->rearranger != rearranger || iodesc->maxregions != 1 ||
iodesc->needsfill || iodesc->mpitype != MPI_INT)
return ERR_WRONG;
for (int e = 0; e < iodesc->maplen; e++)
if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1)
return ERR_WRONG;
if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN ||
iodesc->dimlen[2] != Z_DIM_LEN)
return ERR_WRONG;
if (rearranger == PIO_REARR_SUBSET)
{
if (iodesc->nrecvs != 1 || iodesc->num_aiotasks != TARGET_NTASKS)
return ERR_WRONG;
}
else
{
/* I haven't figured out yet what these should be for
* the box rearranger. */
/* printf("iodesc->nrecv = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs, */
/* iodesc->num_aiotasks); */
}
}
/* Free the PIO decomposition. */
if ((ret = PIOc_freedecomp(iosysid, ioid2)))
ERR(ret);
}
return PIO_NOERR;
}
int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
ios = pio_get_iosystem_from_id(iosysid);
file = (file_desc_t *) malloc(sizeof(file_desc_t));
file->next = NULL;
file->iosystem = ios;
file->iotype = *iotype;
file->buffer.validvars=0;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.vid=NULL;
file->buffer.ioid=-1;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
msg = PIO_MSG_CREATE_FILE;
file->mode = mode;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
// The 64 bit options are not compatable with hdf5 format files
// printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO);
file->mode = file->mode | NC_MPIIO | NC_NETCDF4;
//printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4);
ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh));
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_create(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
if(ierr == PIO_NOERR){
if(ios->io_rank==0){
printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT);
}
int oldfill;
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
// ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill);
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr == PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Create file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
/**
* @ingroup PIO_def_var
* Get current file chunk cache settings from HDF5.
*
* This function has no effect on netCDF classic files. Calling this
* function with iotype of PIO_IOTYPE_PNETCDF or PIO_IOTYPE_NETCDF
* returns an error.
*
* The file chunk cache for HDF5 can be set, and will apply for any
* files opened or created until the program ends, or the settings are
* changed again. The cache settings apply only to the open file. They
* do not persist with the file, and must be set each time the file is
* opened, before it is opened, if they are to have effect.
*
* See the <a
* href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
* variable documentation</a> for details about the operation of this
* function.
*
* Chunksizes have important performance repercussions. NetCDF
* attempts to choose sensible chunk sizes by default, but for best
* performance check chunking against access patterns.
*
* @param iotype the iotype of files to be created or opened.
* @param sizep gets the size of file cache.
* @param nelemsp gets the number of elements in file cache.
* @param preemptionp gets the preemption setting for file cache.
*
* @return PIO_NOERR for success, otherwise an error code.
*/
int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep,
PIO_Offset *nelemsp, float *preemptionp)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
char *errstr;
errstr = NULL;
ierr = PIO_NOERR;
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
/* Since this is a property of the running HDF5 instance, not the
* file, it's not clear if this message passing will apply. For
* now, comment it out. EJH */
/* msg = PIO_MSG_INQ_VAR_FLETCHER32; */
/* if (ios->async_interface && ! ios->ioproc){ */
/* if (ios->compmaster) */
/* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */
/* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */
/* } */
switch (iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
break;
case PIO_IOTYPE_NETCDF4C:
if (!ios->io_rank)
ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
break;
#endif
case PIO_IOTYPE_NETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
default:
ierr = iotype_error(iotype,__FILE__,__LINE__);
}
/* Check for netCDF error. */
MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);
if (!ierr)
{
if (sizep)
if ((ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
ierr = PIO_EIO;
if (nelemsp && !ierr)
if ((ierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
ierr = PIO_EIO;
if (preemptionp && !ierr)
if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
ierr = PIO_EIO;
}
return ierr;
}
/** @ingroup PIO_finalize
* Clean up internal data structures, free MPI resources, and exit the
* pio library.
*
* @param iosysid: the io system ID provided by PIOc_Init_Intracomm().
*
* @returns 0 for success or non-zero for error.
*/
int PIOc_finalize(const int iosysid)
{
iosystem_desc_t *ios, *nios;
int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */
int ierr = PIO_NOERR;
LOG((1, "PIOc_finalize iosysid = %d MPI_COMM_NULL = %d", iosysid, MPI_COMM_NULL));
/* Find the IO system information. */
if (!(ios = pio_get_iosystem_from_id(iosysid)))
return PIO_EBADID;
LOG((3, "found iosystem info comproot = %d union_comm = %d comp_idx = %d",
ios->comproot, ios->union_comm, ios->comp_idx));
/* If asynch IO is in use, send the PIO_MSG_EXIT message from the
* comp master to the IO processes. This may be called by
* componets for other components iosysid. So don't send unless
* there is a valid union_comm. */
if (ios->async_interface && ios->union_comm != MPI_COMM_NULL)
{
int msg = PIO_MSG_EXIT;
LOG((3, "async"));
if (!ios->ioproc)
{
LOG((2, "sending msg = %d ioroot = %d union_comm = %d", msg,
ios->ioroot, ios->union_comm));
/* Send the message to the message handler. */
if (ios->compmaster)
mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
LOG((2, "sending iosysid = %d", iosysid));
/* Send the parameters of the function call. */
if (!mpierr)
mpierr = MPI_Bcast((int *)&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
}
/* Handle MPI errors. */
LOG((3, "handling async errors mpierr = %d my_comm = %d", mpierr, ios->my_comm));
if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
return check_mpi(NULL, mpierr2, __FILE__, __LINE__);
if (mpierr)
return check_mpi(NULL, mpierr, __FILE__, __LINE__);
LOG((3, "async errors bcast"));
}
/* Free this memory that was allocated in init_intracomm. */
if (ios->ioranks)
free(ios->ioranks);
LOG((3, "Freed ioranks."));
/* Free the buffer pool. */
free_cn_buffer_pool(*ios);
LOG((2, "Freed buffer pool."));
/* Free the MPI groups. */
if (ios->compgroup != MPI_GROUP_NULL)
MPI_Group_free(&ios->compgroup);
if (ios->iogroup != MPI_GROUP_NULL)
{
MPI_Group_free(&(ios->iogroup));
LOG((2, "Freed MPI groups."));
}
/* Free the MPI communicators. my_comm is just a copy (but not an
* MPI copy), so does not have to have an MPI_Comm_free()
* call. comp_comm and io_comm are MPI duplicates of the comms
* handed into init_intercomm. So they need to be freed by MPI. */
if (ios->intercomm != MPI_COMM_NULL)
{
LOG((3, "freeing intercomm %d", ios->intercomm));
MPI_Comm_free(&ios->intercomm);
}
if (ios->union_comm != MPI_COMM_NULL)
{
LOG((3, "freeing union_comm %d", ios->union_comm));
MPI_Comm_free(&ios->union_comm);
}
if (ios->io_comm != MPI_COMM_NULL)
{
LOG((3, "freeing io_comm %d", ios->io_comm));
MPI_Comm_free(&ios->io_comm);
}
if (ios->comp_comm != MPI_COMM_NULL)
{
LOG((3, "freeing comp_comm %d", ios->comp_comm));
MPI_Comm_free(&ios->comp_comm);
}
if (ios->my_comm != MPI_COMM_NULL)
ios->my_comm = MPI_COMM_NULL;
/* Delete the iosystem_desc_t data associated with this id. */
LOG((2, "About to delete iosysid %d.", iosysid));
ierr = pio_delete_iosystem_from_list(iosysid);
LOG((2, "Deleted iosysid %d ierr = %d", iosysid, ierr));
LOG((2, "PIOc_finalize completed successfully"));
return ierr;
}
int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
msg = PIO_MSG_OPEN_FILE;
ios = pio_get_iosystem_from_id(iosysid);
if(ios==NULL){
printf("bad iosysid %d\n",iosysid);
return PIO_EBADID;
}
file = (file_desc_t *) malloc(sizeof(*file));
if(file==NULL){
return PIO_ENOMEM;
}
file->iotype = *iotype;
file->next = NULL;
file->iosystem = ios;
file->mode = mode;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
file->buffer.validvars=0;
file->buffer.vid=NULL;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
#ifdef _MPISERIAL
ierr = nc_open(filename, file->mode, &(file->fh));
#else
file->mode = file->mode | NC_MPIIO;
ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh));
#endif
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
// *** Note the INTENTIONAL FALLTHROUGH ***
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
// This should only be done with a file opened to append
if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){
if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT);
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
break;
}
// If we failed to open a file due to an incompatible type of NetCDF, try it
// once with just plain old basic NetCDF
#ifdef _NETCDF
if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) {
if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n");
// reset ierr on all tasks
ierr = PIO_NOERR;
// reset file markers for NETCDF on all tasks
file->iotype = PIO_IOTYPE_NETCDF;
// open netcdf file serially on main task
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh)); }
}
#endif
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr==PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[],
const int maplen, const PIO_Offset *compmap, int *ioidp,const int *rearranger,
const PIO_Offset *iostart,const PIO_Offset *iocount)
{
iosystem_desc_t *ios;
io_desc_t *iodesc;
int mpierr;
int ierr;
int iosize;
int ndisp;
for(int i=0;i<ndims;i++){
if(dims[i]<=0){
piodie("Invalid dims argument",__FILE__,__LINE__);
}
}
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
if(PIO_Save_Decomps){
char filename[30];
if(ios->num_comptasks < 100) {
sprintf(filename, "piodecomp%2.2dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter);
}else if(ios->num_comptasks < 10000) {
sprintf(filename, "piodecomp%4.4dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter);
}else{
sprintf(filename, "piodecomp%6.6dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter);
}
PIOc_writemap(filename,ndims,dims,maplen,compmap,ios->comp_comm);
counter++;
}
iodesc = malloc_iodesc(basetype, ndims);
if(rearranger == NULL)
iodesc->rearranger = ios->default_rearranger;
else
iodesc->rearranger = *rearranger;
if(iodesc->rearranger==PIO_REARR_SUBSET){
if((iostart != NULL) && (iocount != NULL)){
fprintf(stderr,"%s %s\n","Iostart and iocount arguments to PIOc_InitDecomp",
"are incompatable with subset rearrange method and will be ignored");
}
iodesc->num_aiotasks = ios->num_iotasks;
ierr = subset_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc);
}else{
if(ios->ioproc){
// Unless the user specifies the start and count for each IO task compute it.
if((iostart != NULL) && (iocount != NULL)){
// printf("iocount[0] = %ld %ld\n",iocount[0], iocount);
iodesc->maxiobuflen=1;
for(int i=0;i<ndims;i++){
iodesc->firstregion->start[i] = iostart[i];
iodesc->firstregion->count[i] = iocount[i];
compute_maxIObuffersize(ios->io_comm, iodesc);
}
iodesc->num_aiotasks = ios->num_iotasks;
}else{
iodesc->num_aiotasks = CalcStartandCount(basetype, ndims, dims,
ios->num_iotasks, ios->io_rank,
iodesc->firstregion->start, iodesc->firstregion->count);
}
compute_maxIObuffersize(ios->io_comm, iodesc);
}
// Depending on array size and io-blocksize the actual number of io tasks used may vary
CheckMPIReturn(MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot,
ios->my_comm),__FILE__,__LINE__);
// Compute the communications pattern for this decomposition
if(iodesc->rearranger==PIO_REARR_BOX){
ierr = box_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc);
}
/*
if(ios->ioproc){
io_region *ioregion = iodesc->firstregion;
while(ioregion != NULL){
for(int i=0;i<ndims;i++)
printf("%s %d i %d dim %d start %ld count %ld\n",__FILE__,__LINE__,i,dims[i],ioregion->start[i],ioregion->count[i]);
ioregion = ioregion->next;
}
}
*/
}
*ioidp = pio_add_to_iodesc_list(iodesc);
performance_tune_rearranger(*ios, iodesc);
return PIO_NOERR;
}
