int PIO_function()
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
char *errstr;
errstr = NULL;
ierr = PIO_NOERR;
file = pio_get_file_from_id(ncid);
if(file == NULL)
return PIO_EBADID;
ios = file->iosystem;
msg = 0;
if(ios->async_interface && ! ios->ioproc){
if(ios->compmaster)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_function();
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_function();
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_function();
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, errstr,__LINE__);
if(errstr != NULL) free(errstr);
return ierr;
}
///
/// PIO interface to nc_function
///
/// This routine is called collectively by all tasks in the communicator ios.union_comm.
///
/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
///
int PIO_function()
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
var_desc_t *vdesc;
PIO_Offset usage;
int *request;
ierr = PIO_NOERR;
file = pio_get_file_from_id(ncid);
if(file == NULL)
return PIO_EBADID;
ios = file->iosystem;
msg = 0;
if(ios->async_interface && ! ios->ioproc){
if(ios->compmaster)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
ierr = nc_function();
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_function();
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
vdesc = file->varlist + varid;
if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
vdesc->request = realloc(vdesc->request,
sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
}
request = vdesc->request+vdesc->nreqs;
if(ios->io_rank==0){
ierr = ncmpi_function();
}else{
*request = PIO_REQ_NULL;
}
vdesc->nreqs++;
flush_output_buffer(file, false, 0);
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
return ierr;
}
/**
* Write one or more arrays with the same IO decomposition to the
* file.
*
* This funciton is similar to PIOc_write_darray(), but allows the
* caller to use their own data buffering (instead of using the
* buffering implemented in PIOc_write_darray()).
*
* When the user calls PIOc_write_darray() one or more times, then
* PIO_write_darray_multi() will be called when the buffer is flushed.
*
* Internally, this function will:
* <ul>
* <li>Find info about file, decomposition, and variable.
* <li>Do a special flush for pnetcdf if needed.
* <li>Allocates a buffer big enough to hold all the data in the
* multi-buffer, for all tasks.
* <li>Calls rearrange_comp2io() to move data from compute to IO
* tasks.
* <li>For parallel iotypes (pnetcdf and netCDF-4 parallel) call
* pio_write_darray_multi_nc().
* <li>For serial iotypes (netcdf classic and netCDF-4 serial) call
* write_darray_multi_serial().
* <li>For subset rearranger, create holegrid to write missing
* data. Then call pio_write_darray_multi_nc() or
* write_darray_multi_serial() to write the holegrid.
* <li>Special buffer flush for pnetcdf.
* </ul>
*
* @param ncid identifies the netCDF file.
* @param varids an array of length nvars containing the variable ids to
* be written.
* @param ioid the I/O description ID as passed back by
* PIOc_InitDecomp().
* @param nvars the number of variables to be written with this
* call.
* @param arraylen the length of the array to be written. This is the
* length of the distrubited array. That is, the length of the portion
* of the data that is on the processor. The same arraylen is used for
* all variables in the call.
* @param array pointer to the data to be written. This is a pointer
* to an array of arrays with the distributed portion of the array
* that is on this processor. There are nvars arrays of data, and each
* array of data contains one record worth of data for that variable.
* @param frame an array of length nvars with the frame or record
* dimension for each of the nvars variables in IOBUF. NULL if this
* iodesc contains non-record vars.
* @param fillvalue pointer an array (of length nvars) of pointers to
* the fill value to be used for missing data.
* @param flushtodisk non-zero to cause buffers to be flushed to disk.
* @return 0 for success, error code otherwise.
* @ingroup PIO_write_darray
* @author Jim Edwards, Ed Hartnett
*/
int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
PIO_Offset arraylen, void *array, const int *frame,
void **fillvalue, bool flushtodisk)
{
iosystem_desc_t *ios; /* Pointer to io system information. */
file_desc_t *file; /* Pointer to file information. */
io_desc_t *iodesc; /* Pointer to IO description information. */
int rlen; /* Total data buffer size. */
var_desc_t *vdesc0; /* First entry in array of var_desc structure for each var. */
int fndims; /* Number of dims in the var in the file. */
int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function calls. */
int ierr; /* Return code. */
/* Get the file info. */
if ((ierr = pio_get_file(ncid, &file)))
return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
ios = file->iosystem;
/* Check inputs. */
if (nvars <= 0 || !varids)
return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
LOG((1, "PIOc_write_darray_multi ncid = %d ioid = %d nvars = %d arraylen = %ld "
"flushtodisk = %d",
ncid, ioid, nvars, arraylen, flushtodisk));
/* Check that we can write to this file. */
if (!file->writable)
return pio_err(ios, file, PIO_EPERM, __FILE__, __LINE__);
/* Get iodesc. */
if (!(iodesc = pio_get_iodesc_from_id(ioid)))
return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET,
"unknown rearranger", __FILE__, __LINE__);
/* Check the types of all the vars. They must match the type of
* the decomposition. */
for (int v = 0; v < nvars; v++)
{
var_desc_t *vdesc;
if ((ierr = get_var_desc(varids[v], &file->varlist, &vdesc)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
if (vdesc->pio_type != iodesc->piotype)
return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
}
/* Get a pointer to the variable info for the first variable. */
if ((ierr = get_var_desc(varids[0], &file->varlist, &vdesc0)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
/* Run these on all tasks if async is not in use, but only on
* non-IO tasks if async is in use. */
if (!ios->async || !ios->ioproc)
{
/* Get the number of dims for this var. */
LOG((3, "about to call PIOc_inq_varndims varids[0] = %d", varids[0]));
if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[0], &fndims)))
return check_netcdf(file, ierr, __FILE__, __LINE__);
LOG((3, "called PIOc_inq_varndims varids[0] = %d fndims = %d", varids[0], fndims));
}
/* If async is in use, and this is not an IO task, bcast the parameters. */
if (ios->async)
{
if (!ios->ioproc)
{
int msg = PIO_MSG_WRITEDARRAYMULTI;
char frame_present = frame ? true : false; /* Is frame non-NULL? */
char fillvalue_present = fillvalue ? true : false; /* Is fillvalue non-NULL? */
int flushtodisk_int = flushtodisk; /* Need this to be int not boolean. */
if (ios->compmaster == MPI_ROOT)
mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
/* Send the function parameters and associated informaiton
* to the msg handler. */
if (!mpierr)
mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&nvars, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast((void *)varids, nvars, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(array, arraylen * iodesc->piotype_size, MPI_CHAR, ios->compmaster,
ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&frame_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
if (!mpierr && frame_present)
mpierr = MPI_Bcast((void *)frame, nvars, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&fillvalue_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
if (!mpierr && fillvalue_present)
mpierr = MPI_Bcast((void *)fillvalue, nvars * iodesc->piotype_size, MPI_CHAR,
ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&flushtodisk_int, 1, MPI_INT, ios->compmaster, ios->intercomm);
LOG((2, "PIOc_write_darray_multi file->pio_ncid = %d nvars = %d ioid = %d arraylen = %d "
"frame_present = %d fillvalue_present = %d flushtodisk = %d", file->pio_ncid, nvars,
ioid, arraylen, frame_present, fillvalue_present, flushtodisk));
}
/* Handle MPI errors. */
if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
return check_mpi(file, mpierr2, __FILE__, __LINE__);
if (mpierr)
return check_mpi(file, mpierr, __FILE__, __LINE__);
/* Share results known only on computation tasks with IO tasks. */
if ((mpierr = MPI_Bcast(&fndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
check_mpi(file, mpierr, __FILE__, __LINE__);
LOG((3, "shared fndims = %d", fndims));
}
/* if the buffer is already in use in pnetcdf we need to flush first */
if (file->iotype == PIO_IOTYPE_PNETCDF && file->iobuf)
if ((ierr = flush_output_buffer(file, 1, 0)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
pioassert(!file->iobuf, "buffer overwrite",__FILE__, __LINE__);
/* Determine total size of aggregated data (all vars/records).
* For netcdf serial writes we collect the data on io nodes and
* then move that data one node at a time to the io master node
* and write (or read). The buffer size on io task 0 must be as
* large as the largest used to accommodate this serial io
* method. */
rlen = 0;
if (iodesc->llen > 0)
rlen = iodesc->maxiobuflen * nvars;
/* Allocate iobuf. */
if (rlen > 0)
{
/* Allocate memory for the buffer for all vars/records. */
if (!(file->iobuf = bget(iodesc->mpitype_size * (size_t)rlen)))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
LOG((3, "allocated %lld bytes for variable buffer", (size_t)rlen * iodesc->mpitype_size));
/* If fill values are desired, and we're using the BOX
* rearranger, insert fill values. */
if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX)
{
LOG((3, "inerting fill values iodesc->maxiobuflen = %d", iodesc->maxiobuflen));
for (int nv = 0; nv < nvars; nv++)
for (int i = 0; i < iodesc->maxiobuflen; i++)
memcpy(&((char *)file->iobuf)[iodesc->mpitype_size * (i + nv * iodesc->maxiobuflen)],
&((char *)fillvalue)[nv * iodesc->mpitype_size], iodesc->mpitype_size);
}
}
else if (file->iotype == PIO_IOTYPE_PNETCDF && ios->ioproc)
{
/* this assures that iobuf is allocated on all iotasks thus
assuring that the flush_output_buffer call above is called
collectively (from all iotasks) */
if (!(file->iobuf = bget(1)))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
LOG((3, "allocated token for variable buffer"));
}
/* Move data from compute to IO tasks. */
if ((ierr = rearrange_comp2io(ios, iodesc, array, file->iobuf, nvars)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
/* Write the darray based on the iotype. */
LOG((2, "about to write darray for iotype = %d", file->iotype));
switch (file->iotype)
{
case PIO_IOTYPE_NETCDF4P:
case PIO_IOTYPE_PNETCDF:
if ((ierr = write_darray_multi_par(file, nvars, fndims, varids, iodesc,
DARRAY_DATA, frame)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
break;
case PIO_IOTYPE_NETCDF4C:
case PIO_IOTYPE_NETCDF:
if ((ierr = write_darray_multi_serial(file, nvars, fndims, varids, iodesc,
DARRAY_DATA, frame)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
break;
default:
return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
}
/* For PNETCDF the iobuf is freed in flush_output_buffer() */
if (file->iotype != PIO_IOTYPE_PNETCDF)
{
/* Release resources. */
if (file->iobuf)
{
LOG((3,"freeing variable buffer in pio_darray"));
brel(file->iobuf);
file->iobuf = NULL;
}
}
/* The box rearranger will always have data (it could be fill
* data) to fill the entire array - that is the aggregate start
* and count values will completely describe one unlimited
* dimension unit of the array. For the subset method this is not
* necessarily the case, areas of missing data may never be
* written. In order to make sure that these areas are given the
* missing value a 'holegrid' is used to describe the missing
* points. This is generally faster than the netcdf method of
* filling the entire array with missing values before overwriting
* those values later. */
if (iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill)
{
LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars,
iodesc->holegridsize, iodesc->needsfill));
pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);
/* Get a buffer. */
if (ios->io_rank == 0)
vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars);
else if (iodesc->holegridsize > 0)
vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars);
/* copying the fill value into the data buffer for the box
* rearranger. This will be overwritten with data where
* provided. */
for (int nv = 0; nv < nvars; nv++)
for (int i = 0; i < iodesc->holegridsize; i++)
memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
&((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);
/* Write the darray based on the iotype. */
switch (file->iotype)
{
case PIO_IOTYPE_PNETCDF:
case PIO_IOTYPE_NETCDF4P:
if ((ierr = write_darray_multi_par(file, nvars, fndims, varids, iodesc,
DARRAY_FILL, frame)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
break;
case PIO_IOTYPE_NETCDF4C:
case PIO_IOTYPE_NETCDF:
if ((ierr = write_darray_multi_serial(file, nvars, fndims, varids, iodesc,
DARRAY_FILL, frame)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
break;
default:
return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
}
/* For PNETCDF fillbuf is freed in flush_output_buffer() */
if (file->iotype != PIO_IOTYPE_PNETCDF)
{
/* Free resources. */
if (vdesc0->fillbuf)
{
brel(vdesc0->fillbuf);
vdesc0->fillbuf = NULL;
}
}
}
/* Flush data to disk for pnetcdf. */
if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF)
if ((ierr = flush_output_buffer(file, flushtodisk, 0)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
return PIO_NOERR;
}
/**
* @ingroup PIO_def_var
* Set chunksizes for a variable.
*
* This function only applies to netCDF-4 files. When used with netCDF
* classic files, the error PIO_ENOTNC4 will be returned.
*
* See the <a
* href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
* variable documentation</a> for details about the operation of this
* function.
*
* Chunksizes have important performance repercussions. NetCDF
* attempts to choose sensible chunk sizes by default, but for best
* performance check chunking against access patterns.
*
* @param ncid the ncid of the open file.
* @param varid the ID of the variable to set chunksizes for.
* @param storage NC_CONTIGUOUS or NC_CHUNKED.
* @param chunksizep an array of chunksizes. Must have a chunksize for
* every variable dimension.
*
* @return PIO_NOERR for success, otherwise an error code.
*/
int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
float preemption)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
char *errstr;
errstr = NULL;
ierr = PIO_NOERR;
if (!(file = pio_get_file_from_id(ncid)))
return PIO_EBADID;
ios = file->iosystem;
msg = PIO_MSG_SET_VAR_CHUNK_CACHE;
if (ios->async_interface && ! ios->ioproc)
{
if (ios->compmaster)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
}
if (ios->ioproc)
{
switch (file->iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
break;
case PIO_IOTYPE_NETCDF4C:
if (!ios->io_rank)
ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
break;
#endif
case PIO_IOTYPE_NETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
/* Allocate an error string if needed. */
if (ierr != PIO_NOERR)
{
errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
sprintf(errstr,"in file %s",__FILE__);
}
/* Check for netCDF error. */
ierr = check_netcdf(file, ierr, errstr,__LINE__);
/* Free the error string if it was allocated. */
if (errstr != NULL)
free(errstr);
return ierr;
}
/**
* @ingroup PIO_inq_var
* Get the variable chunk cache settings.
*
* This function only applies to netCDF-4 files. When used with netCDF
* classic files, the error PIO_ENOTNC4 will be returned.
*
* Note that these settings are not part of the data file - they apply
* only to the open file as long as it is open.
*
* See the <a
* href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
* variable documentation</a> for details about the operation of this
* function.
*
* @param ncid the ncid of the open file.
* @param varid the ID of the variable to set chunksizes for.
* @param sizep will get the size of the cache in bytes.
* @param nelemsp will get the number of elements in the cache.
* @param preemptionp will get the cache preemption value.
*
* @return PIO_NOERR for success, otherwise an error code.
*/
int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
float *preemptionp)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
char *errstr;
errstr = NULL;
ierr = PIO_NOERR;
if (!(file = pio_get_file_from_id(ncid)))
return PIO_EBADID;
ios = file->iosystem;
/* Since this is a property of the running HDF5 instance, not the
* file, it's not clear if this message passing will apply. For
* now, comment it out. EJH */
/* msg = PIO_MSG_INQ_VAR_FLETCHER32; */
/* if (ios->async_interface && ! ios->ioproc){ */
/* if (ios->compmaster) */
/* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */
/* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */
/* } */
if (ios->ioproc)
{
switch (file->iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_get_var_chunk_cache(file->fh, varid, (size_t *)sizep, (size_t *)nelemsp,
preemptionp);
break;
case PIO_IOTYPE_NETCDF4C:
if (ios->io_rank == 0)
ierr = nc_get_var_chunk_cache(file->fh, varid, (size_t *)sizep, (size_t *)nelemsp,
preemptionp);
break;
#endif
case PIO_IOTYPE_NETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
/* If there is an error, allocate space for the error string. */
if (ierr != PIO_NOERR)
{
errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
sprintf(errstr,"in file %s",__FILE__);
}
/* Check the netCDF return code, and broadcast it to all tasks. */
ierr = check_netcdf(file, ierr, errstr,__LINE__);
/* Free the error string if it was allocated. */
if (errstr != NULL)
free(errstr);
/* Broadcast results to all tasks. */
if (sizep && !ierr)
ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
if (nelemsp && !ierr)
ierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
if (preemptionp && !ierr)
ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm);
return ierr;
}
/**
* @ingroup PIO_def_var
* Set deflate (zlib) settings for a variable.
*
* This function only applies to netCDF-4 files. When used with netCDF
* classic files, the error PIO_ENOTNC4 will be returned.
*
* See the <a
* href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
* variable documentation</a> for details about the operation of this
* function.
*
* @param ncid the ncid of the open file.
* @param varid the ID of the variable.
* @param shuffle non-zero to turn on shuffle filter (can be good for
* integer data).
* @param deflate non-zero to turn on zlib compression for this
* variable.
* @param deflate_level 1 to 9, with 1 being faster and 9 being more
* compressed.
*
* @return PIO_NOERR for success, otherwise an error code.
*/
int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
int deflate_level)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
char *errstr;
errstr = NULL;
ierr = PIO_NOERR;
if (!(file = pio_get_file_from_id(ncid)))
return PIO_EBADID;
ios = file->iosystem;
msg = PIO_MSG_DEF_VAR_DEFLATE;
if (ios->async_interface && ! ios->ioproc)
{
if (ios->compmaster)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
}
if (ios->ioproc)
{
switch (file->iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
/* Versions of netCDF 4.4 and earlier do not return an
* error when attempting to turn on deflation with
* parallel I.O. But this is not allowed by HDF5. So
* return the correct error code. */
ierr = NC_EINVAL;
//ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
break;
case PIO_IOTYPE_NETCDF4C:
if (!ios->io_rank)
ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
break;
#endif
case PIO_IOTYPE_NETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
/* If there is an error, allocate space for the error string. */
if (ierr != PIO_NOERR)
{
errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
sprintf(errstr,"in file %s",__FILE__);
}
/* Check the netCDF return code, and broadcast it to all tasks. */
ierr = check_netcdf(file, ierr, errstr,__LINE__);
/* Free the error string if it was allocated. */
if (errstr != NULL)
free(errstr);
return ierr;
}
/**
* @ingroup PIO_inq_var
* Inquire about chunksizes for a variable.
*
* This function only applies to netCDF-4 files. When used with netCDF
* classic files, the error PIO_ENOTNC4 will be returned.
*
* See the <a
* href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
* variable documentation</a> for details about the operation of this
* function.
*
* @param ncid the ncid of the open file.
* @param varid the ID of the variable to set chunksizes for.
* @param storagep pointer to int which will be set to either
* NC_CONTIGUOUS or NC_CHUNKED.
* @param chunksizep pointer to memory where chunksizes will be
* set. There are the same number of chunksizes as there are
* dimensions.
*
* @return PIO_NOERR for success, otherwise an error code.
*/
int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
char *errstr;
errstr = NULL;
ierr = PIO_NOERR;
if (!(file = pio_get_file_from_id(ncid)))
return PIO_EBADID;
ios = file->iosystem;
msg = PIO_MSG_INQ_VAR_CHUNKING;
if (ios->async_interface && ! ios->ioproc)
{
if (ios->compmaster)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
}
if (ios->ioproc)
{
switch (file->iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_inq_var_endian(file->fh, varid, endianp);
break;
case PIO_IOTYPE_NETCDF4C:
if (!ios->io_rank)
ierr = nc_inq_var_endian(file->fh, varid, endianp);
break;
#endif
case PIO_IOTYPE_NETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = PIO_ENOTNC4;
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
/* If there is an error, allocate space for the error string. */
if (ierr != PIO_NOERR)
{
errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
sprintf(errstr,"in file %s",__FILE__);
}
/* Check the netCDF return code, and broadcast it to all tasks. */
ierr = check_netcdf(file, ierr, errstr,__LINE__);
/* Free the error string if it was allocated. */
if (errstr != NULL)
free(errstr);
/* Broadcast results to all tasks. */
if (endianp)
ierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm);
return ierr;
}
/**
* @name PIOc_sync
*/
int PIOc_sync (int ncid)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
wmulti_buffer *wmb, *twmb;
ierr = PIO_NOERR;
file = pio_get_file_from_id(ncid);
if(file == NULL)
return PIO_EBADID;
ios = file->iosystem;
msg = PIO_MSG_SYNC;
if(ios->async_interface && ! ios->ioproc){
if(ios->compmaster)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
}
if((file->mode & PIO_WRITE)){
// cn_buffer_report( *ios, true);
wmb = &(file->buffer);
while(wmb != NULL){
// printf("%s %d %d %d\n",__FILE__,__LINE__,wmb->ioid, wmb->validvars);
if(wmb->validvars>0){
flush_buffer(ncid, wmb, true);
}
twmb = wmb;
wmb = wmb->next;
if(twmb == &(file->buffer)){
twmb->ioid=-1;
twmb->next=NULL;
}else{
brel(twmb);
}
}
flush_output_buffer(file, true, 0);
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_sync(file->fh);;
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_sync(file->fh);;
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_sync(file->fh);;
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
}
return ierr;
}
/**
** @ingroup PIO_closefile
** @brief close a file previously opened with PIO
** @param ncid: the file pointer
*/
int PIOc_closefile(int ncid)
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
file = pio_get_file_from_id(ncid);
if(file == NULL)
return PIO_EBADID;
ios = file->iosystem;
msg = 0;
if((file->mode & PIO_WRITE)){
PIOc_sync(ncid);
}
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_close(file->fh);
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_close(file->fh);
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
if((file->mode & PIO_WRITE)){
ierr = ncmpi_buffer_detach(file->fh);
}
ierr = ncmpi_close(file->fh);
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
if(ios->io_rank==0){
printf("Close file %d \n",file->fh);
// if(file->fh==5) print_trace(stdout);
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
int iret = pio_delete_file_from_list(ncid);
return ierr;
}
int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
ios = pio_get_iosystem_from_id(iosysid);
file = (file_desc_t *) malloc(sizeof(file_desc_t));
file->next = NULL;
file->iosystem = ios;
file->iotype = *iotype;
file->buffer.validvars=0;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.vid=NULL;
file->buffer.ioid=-1;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
msg = PIO_MSG_CREATE_FILE;
file->mode = mode;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
// The 64 bit options are not compatable with hdf5 format files
// printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO);
file->mode = file->mode | NC_MPIIO | NC_NETCDF4;
//printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4);
ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh));
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_create(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
if(ierr == PIO_NOERR){
if(ios->io_rank==0){
printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT);
}
int oldfill;
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
// ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill);
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr == PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Create file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
msg = PIO_MSG_OPEN_FILE;
ios = pio_get_iosystem_from_id(iosysid);
if(ios==NULL){
printf("bad iosysid %d\n",iosysid);
return PIO_EBADID;
}
file = (file_desc_t *) malloc(sizeof(*file));
if(file==NULL){
return PIO_ENOMEM;
}
file->iotype = *iotype;
file->next = NULL;
file->iosystem = ios;
file->mode = mode;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
file->buffer.validvars=0;
file->buffer.vid=NULL;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
#ifdef _MPISERIAL
ierr = nc_open(filename, file->mode, &(file->fh));
#else
file->mode = file->mode | NC_MPIIO;
ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh));
#endif
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
// *** Note the INTENTIONAL FALLTHROUGH ***
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
// This should only be done with a file opened to append
if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){
if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT);
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
break;
}
// If we failed to open a file due to an incompatible type of NetCDF, try it
// once with just plain old basic NetCDF
#ifdef _NETCDF
if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) {
if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n");
// reset ierr on all tasks
ierr = PIO_NOERR;
// reset file markers for NETCDF on all tasks
file->iotype = PIO_IOTYPE_NETCDF;
// open netcdf file serially on main task
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh)); }
}
#endif
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr==PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
int PIO_function()
{
int ierr;
int msg;
int mpierr;
iosystem_desc_t *ios;
file_desc_t *file;
MPI_Datatype ibuftype;
int ndims;
int ibufcnt;
bool bcast = false;
file = pio_get_file_from_id(ncid);
if(file == NULL)
return PIO_EBADID;
ios = file->iosystem;
msg = 0;
ierr = PIO_NOERR;
if(ios->async_interface && ! ios->ioproc){
if(ios->compmaster)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_function();
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
bcast = true;
if(ios->iomaster){
ierr = nc_function();
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_function();
ierr = ncmpi_function_all();
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ios->async_interface || bcast ||
(ios->num_iotasks < ios->num_comptasks)){
MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
}
return ierr;
}
