/*ARGSUSED*/
int
NCedgeck(const NC *ncp,
const NC_var *varp,
const MPI_Offset *start,
const MPI_Offset *edges)
{
const MPI_Offset *const end = start + varp->ndims;
const MPI_Offset *shp = varp->shape;
if (varp->ndims == 0)
return NC_NOERR; /* 'scalar' variable */
if (IS_RECVAR(varp)) {
start++;
edges++;
shp++;
}
for (; start < end; start++, edges++, shp++) {
if ( (*shp < 0) || (*edges > *shp) || (*start + *edges > *shp))
return(NC_EEDGE);
}
return NC_NOERR;
}
int ncmpi_print_all_var_offsets(int ncid) {
int i;
NC_var **vpp;
NC *ncp;
ncmpii_NC_check_id(ncid, &ncp);
if (ncp->begin_var%1048576)
printf("%s header size (ncp->begin_var)=%lld MB + %lld\n",
ncp->nciop->path, ncp->begin_var/1048575, ncp->begin_var%1048576);
else
printf("%s header size (ncp->begin_var)=%lld MB\n",
ncp->nciop->path, ncp->begin_var/1048575);
vpp = ncp->vars.value;
for (i=0; i<ncp->vars.ndefined; i++, vpp++) {
char str[1024];
MPI_Offset off = (*vpp)->begin;
MPI_Offset rem = off % 1048576;;
if (IS_RECVAR(*vpp))
sprintf(str," Record variable \"%20s\": ",(*vpp)->name->cp);
else
sprintf(str,"non-record variable \"%20s\": ",(*vpp)->name->cp);
if (rem)
printf("%s offset=%12lld MB + %7lld len=%lld\n", str, off/1048576, rem,(*vpp)->len);
else
printf("%s offset=%12lld MB len=%lld\n", str, off/1048576,(*vpp)->len);
}
return NC_NOERR;
}
/*
* Initialize the 'non-record' variables.
*/
static int
fillerup(NC *ncp)
{
int status = NC_NOERR;
size_t ii;
NC_var **varpp;
assert(!NC_readonly(ncp));
assert(NC_dofill(ncp));
/* loop thru vars */
varpp = ncp->vars.value;
for(ii = 0; ii < ncp->vars.nelems; ii++, varpp++)
{
if(IS_RECVAR(*varpp))
{
/* skip record variables */
continue;
}
status = fill_NC_var(ncp, *varpp, (*varpp)->len, 0);
if(status != NC_NOERR)
break;
}
return status;
}
static int
fill_added_recs(NC *gnu, NC *old)
{
NC_var ** const gnu_varpp = (NC_var **)gnu->vars.value;
const int old_nrecs = (int) NC_get_numrecs(old);
int recno = 0;
for(; recno < old_nrecs; recno++)
{
int varid = (int)old->vars.nelems;
for(; varid < (int)gnu->vars.nelems; varid++)
{
const NC_var *const gnu_varp = *(gnu_varpp + varid);
if(!IS_RECVAR(gnu_varp))
{
/* skip non-record variables */
continue;
}
/* else */
{
const int status = fill_NC_var(gnu, gnu_varp, recno);
if(status != NC_NOERR)
return status;
}
}
}
return NC_NOERR;
}
static int
fill_added_recs(NC *gnu, NC *old)
{
NC_var ** const gnu_varpp = (NC_var **)gnu->vars.value;
NC_var *const *const gnu_end = &gnu_varpp[gnu->vars.nelems];
const int old_nrecs = (int) NC_get_numrecs(old);
int recno = 0;
NC_var **vpp = gnu_varpp;
NC_var *const *const end = &vpp[gnu->vars.nelems];
int numrecvars = 0;
NC_var *recvarp = NULL;
/* Determine if there is only one record variable. If so, we
must treat as a special case because there's no record padding */
for(; vpp < end; vpp++) {
if(IS_RECVAR(*vpp)) {
recvarp = *vpp;
numrecvars++;
}
}
for(; recno < old_nrecs; recno++)
{
int varid = (int)old->vars.nelems;
for(; varid < (int)gnu->vars.nelems; varid++)
{
const NC_var *const gnu_varp = *(gnu_varpp + varid);
if(!IS_RECVAR(gnu_varp))
{
/* skip non-record variables */
continue;
}
/* else */
{
size_t varsize = numrecvars == 1 ? gnu->recsize : gnu_varp->len;
const int status = fill_NC_var(gnu, gnu_varp, varsize, recno);
if(status != NC_NOERR)
return status;
}
}
}
return NC_NOERR;
}
/*
* Move the records "out".
* Fill as needed.
*/
static int
move_recs_r(NC *gnu, NC *old)
{
int status;
int recno;
int varid;
NC_var **gnu_varpp = (NC_var **)gnu->vars.value;
NC_var **old_varpp = (NC_var **)old->vars.value;
NC_var *gnu_varp;
NC_var *old_varp;
off_t gnu_off;
off_t old_off;
const size_t old_nrecs = NC_get_numrecs(old);
/* Don't parallelize this loop */
for(recno = (int)old_nrecs -1; recno >= 0; recno--)
{
/* Don't parallelize this loop */
for(varid = (int)old->vars.nelems -1; varid >= 0; varid--)
{
gnu_varp = *(gnu_varpp + varid);
if(!IS_RECVAR(gnu_varp))
{
/* skip non-record variables on this pass */
continue;
}
/* else */
/* else, a pre-existing variable */
old_varp = *(old_varpp + varid);
gnu_off = gnu_varp->begin + (off_t)(gnu->recsize * recno);
old_off = old_varp->begin + (off_t)(old->recsize * recno);
if(gnu_off == old_off)
continue; /* nothing to do */
assert(gnu_off > old_off);
status = gnu->nciop->move(gnu->nciop, gnu_off, old_off,
old_varp->len, 0);
if(status != NC_NOERR)
return status;
}
}
NC_set_numrecs(gnu, old_nrecs);
return NC_NOERR;
}
/*
* Check whether variable size is less than or equal to vlen_max,
* without overflowing in arithmetic calculations. If OK, return 1,
* else, return 0. For CDF1 format or for CDF2 format on non-LFS
* platforms, vlen_max should be 2^31 - 4, but for CDF2 format on
* systems with LFS it should be 2^32 - 4.
*/
int
ncmpii_NC_check_vlen(NC_var *varp, MPI_Offset vlen_max) {
MPI_Offset prod=varp->xsz; /* product of xsz and dimensions so far */
int ii;
for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) {
if (varp->shape[ii] > vlen_max / prod) {
return 0; /* size in bytes won't fit in a 32-bit int */
}
prod *= varp->shape[ii];
}
return 1; /* OK */
}
/*
* Check whether variable size is less than or equal to vlen_max,
* without overflowing in arithmetic calculations. If OK, return 1,
* else, return 0. For CDF1 format or for CDF2 format on non-LFS
* platforms, vlen_max should be 2^31 - 4, but for CDF2 format on
* systems with LFS it should be 2^32 - 4.
*/
int
NC_check_vlen(NC_var *varp, size_t vlen_max) {
size_t prod=varp->xsz; /* product of xsz and dimensions so far */
int ii;
assert(varp != NULL);
for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) {
if (varp->shape[ii] > vlen_max / prod) {
return 0; /* size in bytes won't fit in a 32-bit int */
}
prod *= varp->shape[ii];
}
return 1; /* OK */
}
/*
* Return actual unpadded length (in bytes) of a variable, which
* doesn't include any extra padding used for alignment. For a record
* variable, this is the length in bytes of one record's worth of that
* variable's data.
*/
static off_t
NC_var_unpadded_len(const NC_var *varp, const NC_dimarray *dims)
{
size_t *shp;
off_t product = 1;
if(varp->ndims != 0) {
for(shp = varp->shape + varp->ndims -1; shp >= varp->shape; shp--) {
if(!(shp == varp->shape && IS_RECVAR(varp)))
product *= *shp;
}
}
product = product * varp->xsz;
return product;
}
/*----< ncmpii_is_request_contiguous() >-------------------------------------*/
int
ncmpii_is_request_contiguous(NC_var *varp,
const MPI_Offset starts[],
const MPI_Offset counts[])
{
/* determine whether the get/put request to this variable using
starts[] and counts[] is contiguous in file */
int i, j, most_sig_dim, ndims=varp->ndims;
/* this variable is a scalar */
if (ndims == 0) return 1;
for (i=0; i<ndims; i++)
if (counts[i] == 0) /* zero length request */
return 1;
most_sig_dim = 0; /* record dimension */
if (IS_RECVAR(varp)) {
/* if there are more than one record variabl, then the record
dimensions, counts[0] must == 1. For now, we assume there
are more than one record variable.
TODO: we need the API ncmpi_inq_rec() as in netcdf 3.6.3
to know how many record variables are defined */
if (counts[0] > 1) return 0;
most_sig_dim = 1;
}
for (i=ndims-1; i>most_sig_dim; i--) {
/* find the first counts[i] that is not the entire dimension */
if (counts[i] < varp->shape[i]) {
/* check dim from i-1, i-2, ..., most_sig_dim and
their counts[] should all be 1 */
for (j=i-1; j>=most_sig_dim; j--) {
if (counts[j] > 1)
return 0;
}
break;
}
else { /* counts[i] == varp->shape[i] */
/* when accessing the entire dimension, starts[i] must be 0 */
if (starts[i] != 0) return 0;
}
}
return 1;
}
/*
* Check whether 'coord' values (indices) are valid for the variable.
* Note that even if the request size is zero, this check is enforced in both
* netCDF and PnetCDF. Otherwise, many test cases under test directoy can fail.
*/
int
NCcoordck(NC *ncp,
const NC_var *varp,
const MPI_Offset *coord)
{
const MPI_Offset *ip;
MPI_Offset *up;
if (varp->ndims == 0)
return NC_NOERR; /* 'scalar' variable */
if (IS_RECVAR(varp)) {
/* if (*coord > X_INT64_T_MAX)
return NC_EINVALCOORDS; *//* sanity check */
if (NC_readonly(ncp) && *coord >= ncp->numrecs) {
if (!NC_doNsync(ncp))
return NC_EINVALCOORDS;
/* else */
{
/* Update from disk and check again */
const int status = ncmpii_read_numrecs(ncp);
if (status != NC_NOERR)
return status;
if (*coord >= ncp->numrecs)
return NC_EINVALCOORDS;
}
}
/* skip checking the record dimension */
ip = coord + 1;
up = varp->shape + 1;
}
else {
ip = coord;
up = varp->shape;
}
for (; ip < coord + varp->ndims; ip++, up++) {
if ( (*ip <0) || (*ip >= *up) )
return NC_EINVALCOORDS;
}
return NC_NOERR;
}
/*
* Move the "non record" variables "out".
* Fill as needed.
*/
static int
move_vars_r(NC *gnu, NC *old)
{
int status;
int varid;
NC_var **gnu_varpp = (NC_var **)gnu->vars.value;
NC_var **old_varpp = (NC_var **)old->vars.value;
NC_var *gnu_varp;
NC_var *old_varp;
off_t gnu_off;
off_t old_off;
/* Don't parallelize this loop */
for(varid = (int)old->vars.nelems -1;
varid >= 0; varid--)
{
gnu_varp = *(gnu_varpp + varid);
if(IS_RECVAR(gnu_varp))
{
/* skip record variables on this pass */
continue;
}
/* else */
old_varp = *(old_varpp + varid);
gnu_off = gnu_varp->begin;
old_off = old_varp->begin;
if(gnu_off == old_off)
continue; /* nothing to do */
assert(gnu_off > old_off);
status = gnu->nciop->move(gnu->nciop, gnu_off, old_off,
old_varp->len, 0);
if(status != NC_NOERR)
return status;
}
return NC_NOERR;
}
/*
* Compute the expected size of the file.
*/
int
NC_calcsize(NC *ncp, off_t *calcsizep)
{
NC_var **vpp = (NC_var **)ncp->vars.value;
NC_var *const *const end = &vpp[ncp->vars.nelems];
NC_var *last_fix = NULL; /* last "non-record" var */
NC_var *last_rec = NULL; /* last "record" var */
/*NC_var *last_var;*/
int status;
int numrecvars = 0; /* number of record variables */
if(ncp->vars.nelems == 0) { /* no non-record variables and
no record variables */
*calcsizep = ncp->xsz; /* size of header */
return NC_NOERR;
}
for( /*NADA*/; vpp < end; vpp++) {
status = NC_var_shape(*vpp, &ncp->dims);
if(status != NC_NOERR)
return status;
if(IS_RECVAR(*vpp)) {
last_rec = *vpp;
numrecvars++;
} else {
last_fix = *vpp;
}
}
if(numrecvars == 0) {
assert(last_fix != NULL);
*calcsizep = last_fix->begin + last_fix->len;
/*last_var = last_fix;*/
} else { /* we have at least one record variable */
*calcsizep = ncp->begin_rec + ncp->numrecs * ncp->recsize;
/*last_var = last_rec;*/
}
return NC_NOERR;
}
/*----< NCstrideedgeck() >---------------------------------------------------*/
int
NCstrideedgeck(const NC *ncp,
const NC_var *varp,
const MPI_Offset *start,
const MPI_Offset *edges,
const MPI_Offset *stride)
{
const MPI_Offset *const end = start + varp->ndims;
const MPI_Offset *shp = varp->shape; /* use MPI_Offset for now :( */
if (varp->ndims == 0)
return NC_NOERR; /* 'scalar' variable */
if (IS_RECVAR(varp)) {
if ( *stride == 0 ) /*|| *stride >= X_INT64_T_MAX)*/
/* cast needed for braindead systems with signed MPI_Offset */
return NC_ESTRIDE;
start++;
edges++;
shp++;
stride++;
}
for (; start < end; start++, edges++, shp++, stride++) {
if ( (*shp < 0) ||
(*edges > *shp) ||
(*edges > 0 && *start+1 + (*edges-1) * *stride > *shp) ||
(*edges == 0 && *start > *shp) )
return(NC_EEDGE);
if ( *stride == 0)/* || *stride >= X_INT64_T_MAX)*/
/* cast needed for braindead systems with signed MPI_Offset */
return NC_ESTRIDE;
}
return NC_NOERR;
}
static int
fill_added(NC *gnu, NC *old)
{
NC_var ** const gnu_varpp = (NC_var **)gnu->vars.value;
int varid = (int)old->vars.nelems;
for(; varid < (int)gnu->vars.nelems; varid++)
{
const NC_var *const gnu_varp = *(gnu_varpp + varid);
if(IS_RECVAR(gnu_varp))
{
/* skip record variables */
continue;
}
/* else */
{
const int status = fill_NC_var(gnu, gnu_varp, gnu_varp->len, 0);
if(status != NC_NOERR)
return status;
}
}
return NC_NOERR;
}
/*
* 'compile' the shape and len of a variable
* Formerly
NC_var_shape(var, dims)
*/
int
NC_var_shape(NC_var *varp, const NC_dimarray *dims)
{
size_t *shp, *op;
off_t *dsp;
int *ip = NULL;
const NC_dim *dimp;
off_t product = 1;
varp->xsz = ncx_szof(varp->type);
if(varp->ndims == 0 || varp->dimids == NULL)
{
goto out;
}
/*
* use the user supplied dimension indices
* to determine the shape
*/
for(ip = varp->dimids, op = varp->shape
; ip < &varp->dimids[varp->ndims]; ip++, op++)
{
if(*ip < 0 || (size_t) (*ip) >= ((dims != NULL) ? dims->nelems : 1) )
return NC_EBADDIM;
dimp = elem_NC_dimarray(dims, (size_t)*ip);
*op = dimp->size;
if(*op == NC_UNLIMITED && ip != varp->dimids)
return NC_EUNLIMPOS;
}
/*
* Compute the dsizes
*/
/* ndims is > 0 here */
for(shp = varp->shape + varp->ndims -1,
dsp = varp->dsizes + varp->ndims -1;
shp >= varp->shape;
shp--, dsp--)
{
/*if(!(shp == varp->shape && IS_RECVAR(varp)))*/
if( shp != NULL && (shp != varp->shape || !IS_RECVAR(varp)))
{
if( ((off_t)(*shp)) <= OFF_T_MAX / product )
{
product *= (*shp > 0 ? *shp : 1);
} else
{
product = OFF_T_MAX ;
}
}
*dsp = product;
}
out :
/* No variable size can be > X_INT64_MAX - 3 */
if (0 == NC_check_vlen(varp, X_INT64_MAX-3)) return NC_EVARSIZE;
/*
* For CDF-1 and CDF-2 formats, the total number of array elements
* cannot exceed 2^32, unless this variable is the last fixed-size
* variable, there is no record variable, and the file starting
* offset of this variable is less than 2GiB.
* This will be checked in NC_check_vlens() during NC_endef()
*/
varp->len = product * varp->xsz;
if (varp->len % 4 > 0)
varp->len += 4 - varp->len % 4; /* round up */
#if 0
arrayp("\tshape", varp->ndims, varp->shape);
arrayp("\tdsizes", varp->ndims, varp->dsizes);
#endif
return NC_NOERR;
}
/*----< ncmpii_vars_create_filetype() >--------------------------------------*/
int
ncmpii_vars_create_filetype(NC *ncp,
NC_var *varp,
const MPI_Offset start[],
const MPI_Offset count[],
const MPI_Offset stride[],
int rw_flag,
MPI_Offset *offset_ptr,
MPI_Datatype *filetype_ptr)
{
int dim, status;
MPI_Offset offset, nelems=1;
MPI_Datatype filetype;
if (stride == NULL)
return ncmpii_vara_create_filetype(ncp, varp, start, count, rw_flag,
offset_ptr, filetype_ptr);
offset = varp->begin;
filetype = MPI_BYTE;
for (dim=0; dim<varp->ndims && stride[dim]==1; dim++) ;
if (dim == varp->ndims)
return ncmpii_vara_create_filetype(ncp, varp, start, count, rw_flag,
offset_ptr, filetype_ptr);
/* New coordinate/edge check to fix NC_EINVALCOORDS bug */
status = NCedgeck(ncp, varp, start, count);
if ((status != NC_NOERR) ||
(rw_flag == READ_REQ && IS_RECVAR(varp) && *start + *count > NC_get_numrecs(ncp)))
{
status = NCcoordck(ncp, varp, start);
if (status != NC_NOERR)
return status;
else
return NC_EEDGE;
}
status = NCstrideedgeck(ncp, varp, start, count, stride);
if (status != NC_NOERR)
return status;
if ( rw_flag == READ_REQ && IS_RECVAR(varp) &&
( (*count > 0 && *start+1 + (*count-1) * *stride > NC_get_numrecs(ncp)) ||
(*count == 0 && *start > NC_get_numrecs(ncp)) ) )
return NC_EEDGE;
for (dim=0; dim<varp->ndims; dim++) nelems *= count[dim];
/* filetype is defined only when varp is not a scalar and
the number of requested elemenst > 0
(varp->ndims == 0 meaning this is a scalar variable)
Otherwise, keep filetype MPI_BYTE
*/
if (varp->ndims > 0 && nelems > 0) {
int ndims;
MPI_Datatype tmptype;
MPI_Offset *blocklens, *blockstride, *blockcount;
ndims = varp->ndims;
blocklens = (MPI_Offset*) NCI_Malloc(3 * ndims * sizeof(MPI_Offset));
blockstride = blocklens + ndims;
blockcount = blockstride + ndims;
tmptype = MPI_BYTE;
blocklens[ndims-1] = varp->xsz;
blockcount[ndims-1] = count[ndims-1];
if (ndims == 1 && IS_RECVAR(varp)) {
check_recsize_too_big(ncp);
blockstride[ndims-1] = stride[ndims-1] * ncp->recsize;
offset += start[ndims - 1] * ncp->recsize;
} else {
blockstride[ndims-1] = stride[ndims-1] * varp->xsz;
offset += start[ndims-1] * varp->xsz;
}
for (dim=ndims-1; dim>=0; dim--) {
#if (MPI_VERSION < 2)
MPI_Type_hvector(blockcount[dim], blocklens[dim], blockstride[dim],
tmptype, &filetype);
#else
MPI_Type_create_hvector(blockcount[dim], blocklens[dim],
blockstride[dim], tmptype, &filetype);
#endif
MPI_Type_commit(&filetype);
if (tmptype != MPI_BYTE)
MPI_Type_free(&tmptype);
tmptype = filetype;
if (dim - 1 >= 0) {
blocklens[dim-1] = 1;
blockcount[dim-1] = count[dim - 1];
if (dim-1 == 0 && IS_RECVAR(varp)) {
blockstride[dim-1] = stride[dim-1] * ncp->recsize;
offset += start[dim-1] * ncp->recsize;
} else {
blockstride[dim-1] = stride[dim-1] * varp->dsizes[dim]
* varp->xsz;
offset += start[dim-1] * varp->dsizes[dim] * varp->xsz;
}
}
}
NCI_Free(blocklens);
}
*offset_ptr = offset;
*filetype_ptr = filetype;
return NC_NOERR;
}
/*
* 'compile' the shape and len of a variable
* Formerly
NC_var_shape(var, dims)
*/
int
NC_var_shape(NC_var *varp, const NC_dimarray *dims)
{
size_t *shp, *op;
off_t *dsp;
int *ip;
const NC_dim *dimp;
off_t product = 1;
varp->xsz = ncx_szof(varp->type);
if(varp->ndims == 0)
{
goto out;
}
/*
* use the user supplied dimension indices
* to determine the shape
*/
for(ip = varp->dimids, op = varp->shape
; ip < &varp->dimids[varp->ndims]; ip++, op++)
{
if(*ip < 0 || (size_t) (*ip) >= ((dims != NULL) ? dims->nelems : 1) )
return NC_EBADDIM;
dimp = elem_NC_dimarray(dims, (size_t)*ip);
*op = dimp->size;
if(*op == NC_UNLIMITED && ip != varp->dimids)
return NC_EUNLIMPOS;
}
/*
* Compute the dsizes
*/
/* ndims is > 0 here */
for(shp = varp->shape + varp->ndims -1,
dsp = varp->dsizes + varp->ndims -1;
shp >= varp->shape;
shp--, dsp--)
{
if(!(shp == varp->shape && IS_RECVAR(varp)))
{
if( (off_t)(*shp) <= OFF_T_MAX / product )
{
product *= *shp;
} else
{
product = OFF_T_MAX ;
}
}
*dsp = product;
}
out :
if( varp->xsz <= (X_UINT_MAX - 1) / product ) /* if integer multiply will not overflow */
{
varp->len = product * varp->xsz;
switch(varp->type) {
case NC_BYTE :
case NC_CHAR :
case NC_SHORT :
if( varp->len%4 != 0 )
{
varp->len += 4 - varp->len%4; /* round up */
/* *dsp += 4 - *dsp%4; */
}
break;
default:
/* already aligned */
break;
}
} else
{ /* OK for last var to be "too big", indicated by this special len */
varp->len = X_UINT_MAX;
}
#if 0
arrayp("\tshape", varp->ndims, varp->shape);
arrayp("\tdsizes", varp->ndims, varp->dsizes);
#endif
return NC_NOERR;
}
/*
* Recompute the shapes of all variables
* Sets ncp->begin_var to start of first variable.
* Sets ncp->begin_rec to start of first record variable.
* Returns -1 on error. The only possible error is a reference
* to a non existent dimension, which could occur for a corrupted
* netcdf file.
*/
static int
NC_computeshapes(NC3_INFO* ncp)
{
NC_var **vpp = (NC_var **)ncp->vars.value;
NC_var *const *const end = &vpp[ncp->vars.nelems];
NC_var *first_var = NULL; /* first "non-record" var */
NC_var *first_rec = NULL; /* first "record" var */
int status;
ncp->begin_var = (off_t) ncp->xsz;
ncp->begin_rec = (off_t) ncp->xsz;
ncp->recsize = 0;
if(ncp->vars.nelems == 0)
return(0);
for( /*NADA*/; vpp < end; vpp++)
{
status = NC_var_shape(*vpp, &ncp->dims);
if(status != ENOERR)
return(status);
if(IS_RECVAR(*vpp))
{
if(first_rec == NULL)
first_rec = *vpp;
if((*vpp)->len == UINT32_MAX &&
fIsSet(ncp->flags, NC_64BIT_OFFSET)) /* Flag for large last record */
ncp->recsize += (*vpp)->dsizes[0] * (*vpp)->xsz;
else
ncp->recsize += (*vpp)->len;
}
else
{
if(first_var == NULL)
first_var = *vpp;
/*
* Overwritten each time thru.
* Usually overwritten in first_rec != NULL clause below.
*/
ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len;
}
}
if(first_rec != NULL)
{
if(ncp->begin_rec > first_rec->begin)
return(NC_ENOTNC); /* not a netCDF file or corrupted */
ncp->begin_rec = first_rec->begin;
/*
* for special case of exactly one record variable, pack value
*/
if(ncp->recsize == first_rec->len)
ncp->recsize = *first_rec->dsizes * first_rec->xsz;
}
if(first_var != NULL)
{
ncp->begin_var = first_var->begin;
}
else
{
ncp->begin_var = ncp->begin_rec;
}
if(ncp->begin_var <= 0 ||
ncp->xsz > (size_t)ncp->begin_var ||
ncp->begin_rec <= 0 ||
ncp->begin_var > ncp->begin_rec)
return(NC_ENOTNC); /* not a netCDF file or corrupted */
return(ENOERR);
}
/*----< ncmpii_get_offset() >------------------------------------------------*/
int
ncmpii_get_offset(NC *ncp,
NC_var *varp,
const MPI_Offset starts[], /* [varp->ndims] */
const MPI_Offset counts[], /* [varp->ndims] */
const MPI_Offset strides[], /* [varp->ndims] */
MPI_Offset *offset_ptr) /* return file offset */
{
/* returns the starting file offset when this variable is get/put
with starts[] */
MPI_Offset offset, *end_off=NULL;
int status, i, ndims;
offset = varp->begin; /* beginning file offset of this variable */
ndims = varp->ndims; /* number of dimensions of this variable */
if (counts != NULL)
end_off = (MPI_Offset*) NCI_Malloc(ndims * sizeof(MPI_Offset));
if (counts != NULL && strides != NULL) {
for (i=0; i<ndims; i++)
end_off[i] = starts[i] + counts[i] * strides[i] - 1;
}
else if (counts != NULL) { /* strides == NULL */
for (i=0; i<ndims; i++)
end_off[i] = starts[i] + counts[i] - 1;
}
else { /* when counts == NULL strides is of no use */
end_off = (MPI_Offset*) starts;
}
status = NCcoordck(ncp, varp, end_off); /* validate end_off[] */
if (status != NC_NOERR) {
#ifdef CDEBUG
printf("ncmpii_get_offset(): NCcoordck() fails\n");
#endif
return status;
}
if (ndims > 0) {
if (IS_RECVAR(varp))
/* no need to check recsize here: if MPI_Offset is only 32 bits we
will have had problems long before here */
offset += end_off[0] * ncp->recsize;
else
offset += end_off[ndims-1] * varp->xsz;
if (ndims > 1) {
if (IS_RECVAR(varp))
offset += end_off[ndims - 1] * varp->xsz;
else
offset += end_off[0] * varp->dsizes[1] * varp->xsz;
for (i=1; i<ndims-1; i++)
offset += end_off[i] * varp->dsizes[i+1] * varp->xsz;
}
}
if (counts != NULL && end_off != NULL)
NCI_Free(end_off);
*offset_ptr = offset;
return NC_NOERR;
}
/*----< ncmpii_vara_create_filetype() >--------------------------------------*/
static int
ncmpii_vara_create_filetype(NC *ncp,
NC_var *varp,
const MPI_Offset *start,
const MPI_Offset *count,
int rw_flag,
MPI_Offset *offset_ptr,
MPI_Datatype *filetype_ptr)
{
int dim, status;
MPI_Offset offset, nelems=1;
MPI_Datatype filetype;
offset = varp->begin;
filetype = MPI_BYTE;
/* New coordinate/edge check to fix NC_EINVALCOORDS bug */
status = NCedgeck(ncp, varp, start, count);
if (status != NC_NOERR ||
(rw_flag == READ_REQ && IS_RECVAR(varp) && *start + *count > NC_get_numrecs(ncp)))
{
status = NCcoordck(ncp, varp, start);
if (status != NC_NOERR)
return status;
else
return NC_EEDGE;
}
/* check if the request is contiguous in file
if yes, there is no need to create a filetype */
if (ncmpii_is_request_contiguous(varp, start, count)) {
status = ncmpii_get_offset(ncp, varp, start, NULL, NULL, &offset);
*offset_ptr = offset;
*filetype_ptr = filetype;
return status;
}
for (dim=0; dim<varp->ndims; dim++) nelems *= count[dim];
/* filetype is defined only when varp is not a scalar and
the number of requested elemenst > 0
(varp->ndims == 0 meaning this is a scalar variable)
Otherwise, keep filetype MPI_BYTE
*/
if (varp->ndims > 0 && nelems > 0) {
int i, ndims, blklens[3], tag=0;
int *shape=NULL, *subcount=NULL, *substart=NULL; /* all in bytes */
MPI_Offset *shape64=NULL, *subcount64=NULL, *substart64=NULL;
MPI_Offset size, disps[3];
MPI_Datatype rectype, types[3], type1;
ndims = varp->ndims;
shape = (int*) NCI_Malloc(3 * ndims * sizeof(int));
subcount = shape + ndims;
substart = subcount + ndims;
/* here, request size has been checked and it must > 0 */
if (IS_RECVAR(varp)) {
subcount[0] = count[0];
substart[0] = 0;
shape[0] = subcount[0];
if (ncp->recsize <= varp->len) {
/* the only record variable */
if (varp->ndims == 1) {
shape[0] *= varp->xsz;
subcount[0] *= varp->xsz;
}
else {
for (dim = 1; dim < ndims-1; dim++) {
shape[dim] = varp->shape[dim];
subcount[dim] = count[dim];
substart[dim] = start[dim];
}
shape[dim] = varp->xsz * varp->shape[dim];
subcount[dim] = varp->xsz * count[dim];
substart[dim] = varp->xsz * start[dim];
}
offset += start[0] * ncp->recsize;
MPI_Type_create_subarray(ndims, shape, subcount, substart,
MPI_ORDER_C, MPI_BYTE, &filetype);
MPI_Type_commit(&filetype);
}
else {
check_recsize_too_big(ncp);
/* more than one record variables */
offset += start[0] * ncp->recsize;
if (varp->ndims == 1) {
#if (MPI_VERSION < 2)
MPI_Type_hvector(subcount[0], varp->xsz, ncp->recsize,
MPI_BYTE, &filetype);
#else
MPI_Type_create_hvector(subcount[0], varp->xsz, ncp->recsize,
MPI_BYTE, &filetype);
#endif
MPI_Type_commit(&filetype);
}
else {
for (dim = 1; dim < ndims-1; dim++) {
shape[dim] = varp->shape[dim];
subcount[dim] = count[dim];
substart[dim] = start[dim];
}
shape[dim] = varp->xsz * varp->shape[dim];
subcount[dim] = varp->xsz * count[dim];
substart[dim] = varp->xsz * start[dim];
MPI_Type_create_subarray(ndims-1, shape+1, subcount+1, substart+1,
MPI_ORDER_C, MPI_BYTE, &rectype);
MPI_Type_commit(&rectype);
#if (MPI_VERSION < 2)
MPI_Type_hvector(subcount[0], 1, ncp->recsize, rectype,
&filetype);
#else
MPI_Type_create_hvector(subcount[0], 1, ncp->recsize, rectype,
&filetype);
#endif
MPI_Type_commit(&filetype);
MPI_Type_free(&rectype);
}
}
}
else { /* non record variable */
tag = 0;
for (dim=0; dim< ndims-1; dim++) {
if (varp->shape[dim] > 2147483647) { /* if shape > 2^31-1 */
tag = 1;
break;
}
}
if ((varp->shape[dim]*varp->xsz) > 2147483647)
tag = 1;
if (tag == 0) {
for (dim = 0; dim < ndims-1; dim++ ) {
shape[dim] = varp->shape[dim];
subcount[dim] = count[dim];
substart[dim] = start[dim];
}
shape[dim] = varp->xsz * varp->shape[dim];
subcount[dim] = varp->xsz * count[dim];
substart[dim] = varp->xsz * start[dim];
MPI_Type_create_subarray(ndims, shape, subcount, substart,
MPI_ORDER_C, MPI_BYTE, &filetype);
MPI_Type_commit(&filetype);
}
else {
shape64 = (MPI_Offset*) NCI_Malloc(3 * ndims * sizeof(MPI_Offset));
subcount64 = shape64 + ndims;
substart64 = subcount64 + ndims;
if (ndims == 1) { // for 64-bit support, added July 23, 2008
shape64[0] = varp->shape[0];
subcount64[0] = count[0];
substart64[0] = start[0];
offset += start[0]*varp->xsz;
MPI_Type_contiguous(subcount64[0]*varp->xsz, MPI_BYTE, &type1);
MPI_Type_commit(&type1);
#if (MPI_VERSION < 2)
MPI_Type_hvector(subcount64[0], varp->xsz, shape64[0]*varp->xsz,
MPI_BYTE, &filetype);
#else
MPI_Type_create_hvector(1, 1, shape64[0]*varp->xsz,
type1, &filetype);
#endif
MPI_Type_commit(&filetype);
MPI_Type_free(&type1);
}
else {
for (dim = 0; dim < ndims-1; dim++ ) {
shape64[dim] = varp->shape[dim];
subcount64[dim] = count[dim];
substart64[dim] = start[dim];
}
shape64[dim] = varp->xsz * varp->shape[dim];
subcount64[dim] = varp->xsz * count[dim];
substart64[dim] = varp->xsz * start[dim];
MPI_Type_hvector(subcount64[dim-1],
subcount64[dim],
varp->xsz * varp->shape[dim],
MPI_BYTE,
&type1);
MPI_Type_commit(&type1);
size = shape[dim];
for (i=dim-2; i>=0; i--) {
size *= shape[i+1];
MPI_Type_hvector(subcount64[i],
1,
size,
type1,
&filetype);
MPI_Type_commit(&filetype);
MPI_Type_free(&type1);
type1 = filetype;
}
disps[1] = substart64[dim];
size = 1;
for (i=dim-1; i>=0; i--) {
size *= shape64[i+1];
disps[1] += size*substart64[i];
}
disps[2] = 1;
for (i=0; i<ndims; i++) disps[2] *= shape64[i];
disps[0] = 0;
blklens[0] = blklens[1] = blklens[2] = 1;
types[0] = MPI_LB;
types[1] = type1;
types[2] = MPI_UB;
MPI_Type_struct(3,
blklens,
(MPI_Aint*) disps,
types,
&filetype);
MPI_Type_free(&type1);
}
NCI_Free(shape64);
}
}
NCI_Free(shape);
}
*offset_ptr = offset;
*filetype_ptr = filetype;
return NC_NOERR;
}
/*
* Recompute the shapes of all variables
* Sets ncp->begin_var to start of first variable.
* Sets ncp->begin_rec to start of first record variable.
* Returns -1 on error. The only possible error is a reference
* to a non existent dimension, which could occur for a corrupted
* netcdf file.
*/
static int
NC_computeshapes(NC *ncp)
{
NC_var **vpp = (NC_var **)ncp->vars.value;
NC_var *const *const end = &vpp[ncp->vars.nelems];
NC_var *first_var = NULL; /* first "non-record" var */
NC_var *first_rec = NULL; /* first "record" var */
int status;
ncp->begin_var = (off_t) ncp->xsz;
ncp->begin_rec = (off_t) ncp->xsz;
ncp->recsize = 0;
if(ncp->vars.nelems == 0)
return(0);
for( /*NADA*/; vpp < end; vpp++)
{
status = NC_var_shape(*vpp, &ncp->dims);
if(status != ENOERR)
return(status);
if(IS_RECVAR(*vpp))
{
if(first_rec == NULL)
first_rec = *vpp;
ncp->recsize += (*vpp)->len;
}
else
{
if(first_var == NULL)
first_var = *vpp;
/*
* Overwritten each time thru.
* Usually overwritten in first_rec != NULL clause below.
*/
ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len;
}
}
if(first_rec != NULL)
{
assert(ncp->begin_rec <= first_rec->begin);
ncp->begin_rec = first_rec->begin;
/*
* for special case of exactly one record variable, pack value
*/
if(ncp->recsize == first_rec->len)
ncp->recsize = *first_rec->dsizes * first_rec->xsz;
}
if(first_var != NULL)
{
ncp->begin_var = first_var->begin;
}
else
{
ncp->begin_var = ncp->begin_rec;
}
assert(ncp->begin_var > 0);
assert(ncp->xsz <= (size_t)ncp->begin_var);
assert(ncp->begin_rec > 0);
assert(ncp->begin_var <= ncp->begin_rec);
return(ENOERR);
}
/*
* 'compile' the shape and len of a variable
* Formerly
NC_var_shape(var, dims)
*/
int
NC_var_shape(NC_var *varp, const NC_dimarray *dims)
{
size_t *shp, *dsp, *op;
int *ip;
const NC_dim *dimp;
size_t product = 1;
varp->xsz = ncx_szof(varp->type);
if(varp->ndims == 0)
{
goto out;
}
/*
* use the user supplied dimension indices
* to determine the shape
*/
for(ip = varp->dimids, op = varp->shape
; ip < &varp->dimids[varp->ndims]; ip++, op++)
{
if(*ip < 0 || (size_t) (*ip) >= ((dims != NULL) ? dims->nelems : 1) )
return NC_EBADDIM;
dimp = elem_NC_dimarray(dims, (size_t)*ip);
*op = dimp->size;
if(*op == NC_UNLIMITED && ip != varp->dimids)
return NC_EUNLIMPOS;
}
/*
* Compute the dsizes
*/
/* ndims is > 0 here */
for(shp = varp->shape + varp->ndims -1,
dsp = varp->dsizes + varp->ndims -1;
shp >= varp->shape;
shp--, dsp--)
{
if(!(shp == varp->shape && IS_RECVAR(varp)))
product *= *shp;
*dsp = product;
}
out :
varp->len = product * varp->xsz;
switch(varp->type) {
case NC_BYTE :
case NC_CHAR :
case NC_SHORT :
if( varp->len%4 != 0 )
{
varp->len += 4 - varp->len%4; /* round up */
/* *dsp += 4 - *dsp%4; */
}
break;
default:
/* already aligned */
break;
}
#if 0
arrayp("\tshape", varp->ndims, varp->shape);
arrayp("\tdsizes", varp->ndims, varp->dsizes);
#endif
return NC_NOERR;
}
/*
* Compute each variable's 'begin' offset,
* update 'begin_rec' as well.
*/
static int
NC_begins(NC *ncp,
size_t h_minfree, size_t v_align,
size_t v_minfree, size_t r_align)
{
size_t ii;
int sizeof_off_t;
off_t index = 0;
NC_var **vpp;
NC_var *last = NULL;
if(v_align == NC_ALIGN_CHUNK)
v_align = ncp->chunk;
if(r_align == NC_ALIGN_CHUNK)
r_align = ncp->chunk;
if (fIsSet(ncp->flags, NC_64BIT_OFFSET)) {
sizeof_off_t = 8;
} else {
sizeof_off_t = 4;
}
ncp->xsz = ncx_len_NC(ncp,sizeof_off_t);
if(ncp->vars.nelems == 0)
return NC_NOERR;
/* only (re)calculate begin_var if there is not sufficient space in header
or start of non-record variables is not aligned as requested by valign */
if (ncp->begin_var < ncp->xsz + h_minfree ||
ncp->begin_var != D_RNDUP(ncp->begin_var, v_align) )
{
index = (off_t) ncp->xsz;
ncp->begin_var = D_RNDUP(index, v_align);
if(ncp->begin_var < index + h_minfree)
{
ncp->begin_var = D_RNDUP(index + (off_t)h_minfree, v_align);
}
}
index = ncp->begin_var;
/* loop thru vars, first pass is for the 'non-record' vars */
vpp = ncp->vars.value;
for(ii = 0; ii < ncp->vars.nelems ; ii++, vpp++)
{
if( IS_RECVAR(*vpp) )
{
/* skip record variables on this pass */
continue;
}
#if 0
fprintf(stderr, " VAR %d %s: %ld\n", ii, (*vpp)->name->cp, (long)index);
#endif
if( sizeof_off_t == 4 && (index > X_OFF_MAX || index < 0) )
{
return NC_EVARSIZE;
}
(*vpp)->begin = index;
index += (*vpp)->len;
}
/* only (re)calculate begin_rec if there is not sufficient
space at end of non-record variables or if start of record
variables is not aligned as requested by r_align */
if (ncp->begin_rec < index + v_minfree ||
ncp->begin_rec != D_RNDUP(ncp->begin_rec, r_align) )
{
ncp->begin_rec = D_RNDUP(index, r_align);
if(ncp->begin_rec < index + v_minfree)
{
ncp->begin_rec = D_RNDUP(index + (off_t)v_minfree, r_align);
}
}
index = ncp->begin_rec;
ncp->recsize = 0;
/* loop thru vars, second pass is for the 'record' vars */
vpp = (NC_var **)ncp->vars.value;
for(ii = 0; ii < ncp->vars.nelems; ii++, vpp++)
{
if( !IS_RECVAR(*vpp) )
{
/* skip non-record variables on this pass */
continue;
}
#if 0
fprintf(stderr, " REC %d %s: %ld\n", ii, (*vpp)->name->cp, (long)index);
#endif
if( sizeof_off_t == 4 && (index > X_OFF_MAX || index < 0) )
{
return NC_EVARSIZE;
}
(*vpp)->begin = index;
index += (*vpp)->len;
/* check if record size must fit in 32-bits */
#if SIZEOF_OFF_T == SIZEOF_SIZE_T && SIZEOF_SIZE_T == 4
if( ncp->recsize > X_UINT_MAX - (*vpp)->len )
{
return NC_EVARSIZE;
}
#endif
ncp->recsize += (*vpp)->len;
last = (*vpp);
}
/*
* for special case of exactly one record variable, pack value
*/
if(last != NULL && ncp->recsize == last->len)
ncp->recsize = *last->dsizes * last->xsz;
if(NC_IsNew(ncp))
NC_set_numrecs(ncp, 0);
return NC_NOERR;
}
/*
* Given a valid ncp, return NC_EVARSIZE if any variable has a bad len
* (product of non-rec dim sizes too large), else return NC_NOERR.
*/
static int
NC_check_vlens(NC *ncp)
{
NC_var **vpp;
/* maximum permitted variable size (or size of one record's worth
of a record variable) in bytes. This is different for format 1
and format 2. */
size_t vlen_max;
size_t ii;
size_t large_vars_count;
size_t rec_vars_count;
int last = 0;
if(ncp->vars.nelems == 0)
return NC_NOERR;
if ((ncp->flags & NC_64BIT_OFFSET) && sizeof(off_t) > 4) {
/* CDF2 format and LFS */
vlen_max = X_UINT_MAX - 3; /* "- 3" handles rounded-up size */
} else {
/* CDF1 format */
vlen_max = X_INT_MAX - 3;
}
/* Loop through vars, first pass is for non-record variables. */
large_vars_count = 0;
rec_vars_count = 0;
vpp = ncp->vars.value;
for (ii = 0; ii < ncp->vars.nelems; ii++, vpp++) {
if( !IS_RECVAR(*vpp) ) {
last = 0;
if( NC_check_vlen(*vpp, vlen_max) == 0 ) {
large_vars_count++;
last = 1;
}
} else {
rec_vars_count++;
}
}
/* OK if last non-record variable size too large, since not used to
compute an offset */
if( large_vars_count > 1) { /* only one "too-large" variable allowed */
return NC_EVARSIZE;
}
/* and it has to be the last one */
if( large_vars_count == 1 && last == 0) {
return NC_EVARSIZE;
}
if( rec_vars_count > 0 ) {
/* and if it's the last one, there can't be any record variables */
if( large_vars_count == 1 && last == 1) {
return NC_EVARSIZE;
}
/* Loop through vars, second pass is for record variables. */
large_vars_count = 0;
vpp = ncp->vars.value;
for (ii = 0; ii < ncp->vars.nelems; ii++, vpp++) {
if( IS_RECVAR(*vpp) ) {
last = 0;
if( NC_check_vlen(*vpp, vlen_max) == 0 ) {
large_vars_count++;
last = 1;
}
}
}
/* OK if last record variable size too large, since not used to
compute an offset */
if( large_vars_count > 1) { /* only one "too-large" variable allowed */
return NC_EVARSIZE;
}
/* and it has to be the last one */
if( large_vars_count == 1 && last == 0) {
return NC_EVARSIZE;
}
}
return NC_NOERR;
}
