/*ARGSUSED*/ int NCedgeck(const NC *ncp, const NC_var *varp, const MPI_Offset *start, const MPI_Offset *edges) { const MPI_Offset *const end = start + varp->ndims; const MPI_Offset *shp = varp->shape; if (varp->ndims == 0) return NC_NOERR; /* 'scalar' variable */ if (IS_RECVAR(varp)) { start++; edges++; shp++; } for (; start < end; start++, edges++, shp++) { if ( (*shp < 0) || (*edges > *shp) || (*start + *edges > *shp)) return(NC_EEDGE); } return NC_NOERR; }
int ncmpi_print_all_var_offsets(int ncid) { int i; NC_var **vpp; NC *ncp; ncmpii_NC_check_id(ncid, &ncp); if (ncp->begin_var%1048576) printf("%s header size (ncp->begin_var)=%lld MB + %lld\n", ncp->nciop->path, ncp->begin_var/1048575, ncp->begin_var%1048576); else printf("%s header size (ncp->begin_var)=%lld MB\n", ncp->nciop->path, ncp->begin_var/1048575); vpp = ncp->vars.value; for (i=0; i<ncp->vars.ndefined; i++, vpp++) { char str[1024]; MPI_Offset off = (*vpp)->begin; MPI_Offset rem = off % 1048576;; if (IS_RECVAR(*vpp)) sprintf(str," Record variable \"%20s\": ",(*vpp)->name->cp); else sprintf(str,"non-record variable \"%20s\": ",(*vpp)->name->cp); if (rem) printf("%s offset=%12lld MB + %7lld len=%lld\n", str, off/1048576, rem,(*vpp)->len); else printf("%s offset=%12lld MB len=%lld\n", str, off/1048576,(*vpp)->len); } return NC_NOERR; }
/* * Initialize the 'non-record' variables. */ static int fillerup(NC *ncp) { int status = NC_NOERR; size_t ii; NC_var **varpp; assert(!NC_readonly(ncp)); assert(NC_dofill(ncp)); /* loop thru vars */ varpp = ncp->vars.value; for(ii = 0; ii < ncp->vars.nelems; ii++, varpp++) { if(IS_RECVAR(*varpp)) { /* skip record variables */ continue; } status = fill_NC_var(ncp, *varpp, (*varpp)->len, 0); if(status != NC_NOERR) break; } return status; }
static int fill_added_recs(NC *gnu, NC *old) { NC_var ** const gnu_varpp = (NC_var **)gnu->vars.value; const int old_nrecs = (int) NC_get_numrecs(old); int recno = 0; for(; recno < old_nrecs; recno++) { int varid = (int)old->vars.nelems; for(; varid < (int)gnu->vars.nelems; varid++) { const NC_var *const gnu_varp = *(gnu_varpp + varid); if(!IS_RECVAR(gnu_varp)) { /* skip non-record variables */ continue; } /* else */ { const int status = fill_NC_var(gnu, gnu_varp, recno); if(status != NC_NOERR) return status; } } } return NC_NOERR; }
static int fill_added_recs(NC *gnu, NC *old) { NC_var ** const gnu_varpp = (NC_var **)gnu->vars.value; NC_var *const *const gnu_end = &gnu_varpp[gnu->vars.nelems]; const int old_nrecs = (int) NC_get_numrecs(old); int recno = 0; NC_var **vpp = gnu_varpp; NC_var *const *const end = &vpp[gnu->vars.nelems]; int numrecvars = 0; NC_var *recvarp = NULL; /* Determine if there is only one record variable. If so, we must treat as a special case because there's no record padding */ for(; vpp < end; vpp++) { if(IS_RECVAR(*vpp)) { recvarp = *vpp; numrecvars++; } } for(; recno < old_nrecs; recno++) { int varid = (int)old->vars.nelems; for(; varid < (int)gnu->vars.nelems; varid++) { const NC_var *const gnu_varp = *(gnu_varpp + varid); if(!IS_RECVAR(gnu_varp)) { /* skip non-record variables */ continue; } /* else */ { size_t varsize = numrecvars == 1 ? gnu->recsize : gnu_varp->len; const int status = fill_NC_var(gnu, gnu_varp, varsize, recno); if(status != NC_NOERR) return status; } } } return NC_NOERR; }
/* * Move the records "out". * Fill as needed. */ static int move_recs_r(NC *gnu, NC *old) { int status; int recno; int varid; NC_var **gnu_varpp = (NC_var **)gnu->vars.value; NC_var **old_varpp = (NC_var **)old->vars.value; NC_var *gnu_varp; NC_var *old_varp; off_t gnu_off; off_t old_off; const size_t old_nrecs = NC_get_numrecs(old); /* Don't parallelize this loop */ for(recno = (int)old_nrecs -1; recno >= 0; recno--) { /* Don't parallelize this loop */ for(varid = (int)old->vars.nelems -1; varid >= 0; varid--) { gnu_varp = *(gnu_varpp + varid); if(!IS_RECVAR(gnu_varp)) { /* skip non-record variables on this pass */ continue; } /* else */ /* else, a pre-existing variable */ old_varp = *(old_varpp + varid); gnu_off = gnu_varp->begin + (off_t)(gnu->recsize * recno); old_off = old_varp->begin + (off_t)(old->recsize * recno); if(gnu_off == old_off) continue; /* nothing to do */ assert(gnu_off > old_off); status = gnu->nciop->move(gnu->nciop, gnu_off, old_off, old_varp->len, 0); if(status != NC_NOERR) return status; } } NC_set_numrecs(gnu, old_nrecs); return NC_NOERR; }
/* * Check whether variable size is less than or equal to vlen_max, * without overflowing in arithmetic calculations. If OK, return 1, * else, return 0. For CDF1 format or for CDF2 format on non-LFS * platforms, vlen_max should be 2^31 - 4, but for CDF2 format on * systems with LFS it should be 2^32 - 4. */ int ncmpii_NC_check_vlen(NC_var *varp, MPI_Offset vlen_max) { MPI_Offset prod=varp->xsz; /* product of xsz and dimensions so far */ int ii; for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) { if (varp->shape[ii] > vlen_max / prod) { return 0; /* size in bytes won't fit in a 32-bit int */ } prod *= varp->shape[ii]; } return 1; /* OK */ }
/* * Check whether variable size is less than or equal to vlen_max, * without overflowing in arithmetic calculations. If OK, return 1, * else, return 0. For CDF1 format or for CDF2 format on non-LFS * platforms, vlen_max should be 2^31 - 4, but for CDF2 format on * systems with LFS it should be 2^32 - 4. */ int NC_check_vlen(NC_var *varp, size_t vlen_max) { size_t prod=varp->xsz; /* product of xsz and dimensions so far */ int ii; assert(varp != NULL); for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) { if (varp->shape[ii] > vlen_max / prod) { return 0; /* size in bytes won't fit in a 32-bit int */ } prod *= varp->shape[ii]; } return 1; /* OK */ }
/* * Return actual unpadded length (in bytes) of a variable, which * doesn't include any extra padding used for alignment. For a record * variable, this is the length in bytes of one record's worth of that * variable's data. */ static off_t NC_var_unpadded_len(const NC_var *varp, const NC_dimarray *dims) { size_t *shp; off_t product = 1; if(varp->ndims != 0) { for(shp = varp->shape + varp->ndims -1; shp >= varp->shape; shp--) { if(!(shp == varp->shape && IS_RECVAR(varp))) product *= *shp; } } product = product * varp->xsz; return product; }
/*----< ncmpii_is_request_contiguous() >-------------------------------------*/ int ncmpii_is_request_contiguous(NC_var *varp, const MPI_Offset starts[], const MPI_Offset counts[]) { /* determine whether the get/put request to this variable using starts[] and counts[] is contiguous in file */ int i, j, most_sig_dim, ndims=varp->ndims; /* this variable is a scalar */ if (ndims == 0) return 1; for (i=0; i<ndims; i++) if (counts[i] == 0) /* zero length request */ return 1; most_sig_dim = 0; /* record dimension */ if (IS_RECVAR(varp)) { /* if there are more than one record variabl, then the record dimensions, counts[0] must == 1. For now, we assume there are more than one record variable. TODO: we need the API ncmpi_inq_rec() as in netcdf 3.6.3 to know how many record variables are defined */ if (counts[0] > 1) return 0; most_sig_dim = 1; } for (i=ndims-1; i>most_sig_dim; i--) { /* find the first counts[i] that is not the entire dimension */ if (counts[i] < varp->shape[i]) { /* check dim from i-1, i-2, ..., most_sig_dim and their counts[] should all be 1 */ for (j=i-1; j>=most_sig_dim; j--) { if (counts[j] > 1) return 0; } break; } else { /* counts[i] == varp->shape[i] */ /* when accessing the entire dimension, starts[i] must be 0 */ if (starts[i] != 0) return 0; } } return 1; }
/* * Check whether 'coord' values (indices) are valid for the variable. * Note that even if the request size is zero, this check is enforced in both * netCDF and PnetCDF. Otherwise, many test cases under test directoy can fail. */ int NCcoordck(NC *ncp, const NC_var *varp, const MPI_Offset *coord) { const MPI_Offset *ip; MPI_Offset *up; if (varp->ndims == 0) return NC_NOERR; /* 'scalar' variable */ if (IS_RECVAR(varp)) { /* if (*coord > X_INT64_T_MAX) return NC_EINVALCOORDS; *//* sanity check */ if (NC_readonly(ncp) && *coord >= ncp->numrecs) { if (!NC_doNsync(ncp)) return NC_EINVALCOORDS; /* else */ { /* Update from disk and check again */ const int status = ncmpii_read_numrecs(ncp); if (status != NC_NOERR) return status; if (*coord >= ncp->numrecs) return NC_EINVALCOORDS; } } /* skip checking the record dimension */ ip = coord + 1; up = varp->shape + 1; } else { ip = coord; up = varp->shape; } for (; ip < coord + varp->ndims; ip++, up++) { if ( (*ip <0) || (*ip >= *up) ) return NC_EINVALCOORDS; } return NC_NOERR; }
/* * Move the "non record" variables "out". * Fill as needed. */ static int move_vars_r(NC *gnu, NC *old) { int status; int varid; NC_var **gnu_varpp = (NC_var **)gnu->vars.value; NC_var **old_varpp = (NC_var **)old->vars.value; NC_var *gnu_varp; NC_var *old_varp; off_t gnu_off; off_t old_off; /* Don't parallelize this loop */ for(varid = (int)old->vars.nelems -1; varid >= 0; varid--) { gnu_varp = *(gnu_varpp + varid); if(IS_RECVAR(gnu_varp)) { /* skip record variables on this pass */ continue; } /* else */ old_varp = *(old_varpp + varid); gnu_off = gnu_varp->begin; old_off = old_varp->begin; if(gnu_off == old_off) continue; /* nothing to do */ assert(gnu_off > old_off); status = gnu->nciop->move(gnu->nciop, gnu_off, old_off, old_varp->len, 0); if(status != NC_NOERR) return status; } return NC_NOERR; }
/* * Compute the expected size of the file. */ int NC_calcsize(NC *ncp, off_t *calcsizep) { NC_var **vpp = (NC_var **)ncp->vars.value; NC_var *const *const end = &vpp[ncp->vars.nelems]; NC_var *last_fix = NULL; /* last "non-record" var */ NC_var *last_rec = NULL; /* last "record" var */ /*NC_var *last_var;*/ int status; int numrecvars = 0; /* number of record variables */ if(ncp->vars.nelems == 0) { /* no non-record variables and no record variables */ *calcsizep = ncp->xsz; /* size of header */ return NC_NOERR; } for( /*NADA*/; vpp < end; vpp++) { status = NC_var_shape(*vpp, &ncp->dims); if(status != NC_NOERR) return status; if(IS_RECVAR(*vpp)) { last_rec = *vpp; numrecvars++; } else { last_fix = *vpp; } } if(numrecvars == 0) { assert(last_fix != NULL); *calcsizep = last_fix->begin + last_fix->len; /*last_var = last_fix;*/ } else { /* we have at least one record variable */ *calcsizep = ncp->begin_rec + ncp->numrecs * ncp->recsize; /*last_var = last_rec;*/ } return NC_NOERR; }
/*----< NCstrideedgeck() >---------------------------------------------------*/ int NCstrideedgeck(const NC *ncp, const NC_var *varp, const MPI_Offset *start, const MPI_Offset *edges, const MPI_Offset *stride) { const MPI_Offset *const end = start + varp->ndims; const MPI_Offset *shp = varp->shape; /* use MPI_Offset for now :( */ if (varp->ndims == 0) return NC_NOERR; /* 'scalar' variable */ if (IS_RECVAR(varp)) { if ( *stride == 0 ) /*|| *stride >= X_INT64_T_MAX)*/ /* cast needed for braindead systems with signed MPI_Offset */ return NC_ESTRIDE; start++; edges++; shp++; stride++; } for (; start < end; start++, edges++, shp++, stride++) { if ( (*shp < 0) || (*edges > *shp) || (*edges > 0 && *start+1 + (*edges-1) * *stride > *shp) || (*edges == 0 && *start > *shp) ) return(NC_EEDGE); if ( *stride == 0)/* || *stride >= X_INT64_T_MAX)*/ /* cast needed for braindead systems with signed MPI_Offset */ return NC_ESTRIDE; } return NC_NOERR; }
static int fill_added(NC *gnu, NC *old) { NC_var ** const gnu_varpp = (NC_var **)gnu->vars.value; int varid = (int)old->vars.nelems; for(; varid < (int)gnu->vars.nelems; varid++) { const NC_var *const gnu_varp = *(gnu_varpp + varid); if(IS_RECVAR(gnu_varp)) { /* skip record variables */ continue; } /* else */ { const int status = fill_NC_var(gnu, gnu_varp, gnu_varp->len, 0); if(status != NC_NOERR) return status; } } return NC_NOERR; }
/* * 'compile' the shape and len of a variable * Formerly NC_var_shape(var, dims) */ int NC_var_shape(NC_var *varp, const NC_dimarray *dims) { size_t *shp, *op; off_t *dsp; int *ip = NULL; const NC_dim *dimp; off_t product = 1; varp->xsz = ncx_szof(varp->type); if(varp->ndims == 0 || varp->dimids == NULL) { goto out; } /* * use the user supplied dimension indices * to determine the shape */ for(ip = varp->dimids, op = varp->shape ; ip < &varp->dimids[varp->ndims]; ip++, op++) { if(*ip < 0 || (size_t) (*ip) >= ((dims != NULL) ? dims->nelems : 1) ) return NC_EBADDIM; dimp = elem_NC_dimarray(dims, (size_t)*ip); *op = dimp->size; if(*op == NC_UNLIMITED && ip != varp->dimids) return NC_EUNLIMPOS; } /* * Compute the dsizes */ /* ndims is > 0 here */ for(shp = varp->shape + varp->ndims -1, dsp = varp->dsizes + varp->ndims -1; shp >= varp->shape; shp--, dsp--) { /*if(!(shp == varp->shape && IS_RECVAR(varp)))*/ if( shp != NULL && (shp != varp->shape || !IS_RECVAR(varp))) { if( ((off_t)(*shp)) <= OFF_T_MAX / product ) { product *= (*shp > 0 ? *shp : 1); } else { product = OFF_T_MAX ; } } *dsp = product; } out : /* No variable size can be > X_INT64_MAX - 3 */ if (0 == NC_check_vlen(varp, X_INT64_MAX-3)) return NC_EVARSIZE; /* * For CDF-1 and CDF-2 formats, the total number of array elements * cannot exceed 2^32, unless this variable is the last fixed-size * variable, there is no record variable, and the file starting * offset of this variable is less than 2GiB. * This will be checked in NC_check_vlens() during NC_endef() */ varp->len = product * varp->xsz; if (varp->len % 4 > 0) varp->len += 4 - varp->len % 4; /* round up */ #if 0 arrayp("\tshape", varp->ndims, varp->shape); arrayp("\tdsizes", varp->ndims, varp->dsizes); #endif return NC_NOERR; }
/*----< ncmpii_vars_create_filetype() >--------------------------------------*/ int ncmpii_vars_create_filetype(NC *ncp, NC_var *varp, const MPI_Offset start[], const MPI_Offset count[], const MPI_Offset stride[], int rw_flag, MPI_Offset *offset_ptr, MPI_Datatype *filetype_ptr) { int dim, status; MPI_Offset offset, nelems=1; MPI_Datatype filetype; if (stride == NULL) return ncmpii_vara_create_filetype(ncp, varp, start, count, rw_flag, offset_ptr, filetype_ptr); offset = varp->begin; filetype = MPI_BYTE; for (dim=0; dim<varp->ndims && stride[dim]==1; dim++) ; if (dim == varp->ndims) return ncmpii_vara_create_filetype(ncp, varp, start, count, rw_flag, offset_ptr, filetype_ptr); /* New coordinate/edge check to fix NC_EINVALCOORDS bug */ status = NCedgeck(ncp, varp, start, count); if ((status != NC_NOERR) || (rw_flag == READ_REQ && IS_RECVAR(varp) && *start + *count > NC_get_numrecs(ncp))) { status = NCcoordck(ncp, varp, start); if (status != NC_NOERR) return status; else return NC_EEDGE; } status = NCstrideedgeck(ncp, varp, start, count, stride); if (status != NC_NOERR) return status; if ( rw_flag == READ_REQ && IS_RECVAR(varp) && ( (*count > 0 && *start+1 + (*count-1) * *stride > NC_get_numrecs(ncp)) || (*count == 0 && *start > NC_get_numrecs(ncp)) ) ) return NC_EEDGE; for (dim=0; dim<varp->ndims; dim++) nelems *= count[dim]; /* filetype is defined only when varp is not a scalar and the number of requested elemenst > 0 (varp->ndims == 0 meaning this is a scalar variable) Otherwise, keep filetype MPI_BYTE */ if (varp->ndims > 0 && nelems > 0) { int ndims; MPI_Datatype tmptype; MPI_Offset *blocklens, *blockstride, *blockcount; ndims = varp->ndims; blocklens = (MPI_Offset*) NCI_Malloc(3 * ndims * sizeof(MPI_Offset)); blockstride = blocklens + ndims; blockcount = blockstride + ndims; tmptype = MPI_BYTE; blocklens[ndims-1] = varp->xsz; blockcount[ndims-1] = count[ndims-1]; if (ndims == 1 && IS_RECVAR(varp)) { check_recsize_too_big(ncp); blockstride[ndims-1] = stride[ndims-1] * ncp->recsize; offset += start[ndims - 1] * ncp->recsize; } else { blockstride[ndims-1] = stride[ndims-1] * varp->xsz; offset += start[ndims-1] * varp->xsz; } for (dim=ndims-1; dim>=0; dim--) { #if (MPI_VERSION < 2) MPI_Type_hvector(blockcount[dim], blocklens[dim], blockstride[dim], tmptype, &filetype); #else MPI_Type_create_hvector(blockcount[dim], blocklens[dim], blockstride[dim], tmptype, &filetype); #endif MPI_Type_commit(&filetype); if (tmptype != MPI_BYTE) MPI_Type_free(&tmptype); tmptype = filetype; if (dim - 1 >= 0) { blocklens[dim-1] = 1; blockcount[dim-1] = count[dim - 1]; if (dim-1 == 0 && IS_RECVAR(varp)) { blockstride[dim-1] = stride[dim-1] * ncp->recsize; offset += start[dim-1] * ncp->recsize; } else { blockstride[dim-1] = stride[dim-1] * varp->dsizes[dim] * varp->xsz; offset += start[dim-1] * varp->dsizes[dim] * varp->xsz; } } } NCI_Free(blocklens); } *offset_ptr = offset; *filetype_ptr = filetype; return NC_NOERR; }
/* * 'compile' the shape and len of a variable * Formerly NC_var_shape(var, dims) */ int NC_var_shape(NC_var *varp, const NC_dimarray *dims) { size_t *shp, *op; off_t *dsp; int *ip; const NC_dim *dimp; off_t product = 1; varp->xsz = ncx_szof(varp->type); if(varp->ndims == 0) { goto out; } /* * use the user supplied dimension indices * to determine the shape */ for(ip = varp->dimids, op = varp->shape ; ip < &varp->dimids[varp->ndims]; ip++, op++) { if(*ip < 0 || (size_t) (*ip) >= ((dims != NULL) ? dims->nelems : 1) ) return NC_EBADDIM; dimp = elem_NC_dimarray(dims, (size_t)*ip); *op = dimp->size; if(*op == NC_UNLIMITED && ip != varp->dimids) return NC_EUNLIMPOS; } /* * Compute the dsizes */ /* ndims is > 0 here */ for(shp = varp->shape + varp->ndims -1, dsp = varp->dsizes + varp->ndims -1; shp >= varp->shape; shp--, dsp--) { if(!(shp == varp->shape && IS_RECVAR(varp))) { if( (off_t)(*shp) <= OFF_T_MAX / product ) { product *= *shp; } else { product = OFF_T_MAX ; } } *dsp = product; } out : if( varp->xsz <= (X_UINT_MAX - 1) / product ) /* if integer multiply will not overflow */ { varp->len = product * varp->xsz; switch(varp->type) { case NC_BYTE : case NC_CHAR : case NC_SHORT : if( varp->len%4 != 0 ) { varp->len += 4 - varp->len%4; /* round up */ /* *dsp += 4 - *dsp%4; */ } break; default: /* already aligned */ break; } } else { /* OK for last var to be "too big", indicated by this special len */ varp->len = X_UINT_MAX; } #if 0 arrayp("\tshape", varp->ndims, varp->shape); arrayp("\tdsizes", varp->ndims, varp->dsizes); #endif return NC_NOERR; }
/* * Recompute the shapes of all variables * Sets ncp->begin_var to start of first variable. * Sets ncp->begin_rec to start of first record variable. * Returns -1 on error. The only possible error is a reference * to a non existent dimension, which could occur for a corrupted * netcdf file. */ static int NC_computeshapes(NC3_INFO* ncp) { NC_var **vpp = (NC_var **)ncp->vars.value; NC_var *const *const end = &vpp[ncp->vars.nelems]; NC_var *first_var = NULL; /* first "non-record" var */ NC_var *first_rec = NULL; /* first "record" var */ int status; ncp->begin_var = (off_t) ncp->xsz; ncp->begin_rec = (off_t) ncp->xsz; ncp->recsize = 0; if(ncp->vars.nelems == 0) return(0); for( /*NADA*/; vpp < end; vpp++) { status = NC_var_shape(*vpp, &ncp->dims); if(status != ENOERR) return(status); if(IS_RECVAR(*vpp)) { if(first_rec == NULL) first_rec = *vpp; if((*vpp)->len == UINT32_MAX && fIsSet(ncp->flags, NC_64BIT_OFFSET)) /* Flag for large last record */ ncp->recsize += (*vpp)->dsizes[0] * (*vpp)->xsz; else ncp->recsize += (*vpp)->len; } else { if(first_var == NULL) first_var = *vpp; /* * Overwritten each time thru. * Usually overwritten in first_rec != NULL clause below. */ ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; } } if(first_rec != NULL) { if(ncp->begin_rec > first_rec->begin) return(NC_ENOTNC); /* not a netCDF file or corrupted */ ncp->begin_rec = first_rec->begin; /* * for special case of exactly one record variable, pack value */ if(ncp->recsize == first_rec->len) ncp->recsize = *first_rec->dsizes * first_rec->xsz; } if(first_var != NULL) { ncp->begin_var = first_var->begin; } else { ncp->begin_var = ncp->begin_rec; } if(ncp->begin_var <= 0 || ncp->xsz > (size_t)ncp->begin_var || ncp->begin_rec <= 0 || ncp->begin_var > ncp->begin_rec) return(NC_ENOTNC); /* not a netCDF file or corrupted */ return(ENOERR); }
/*----< ncmpii_get_offset() >------------------------------------------------*/ int ncmpii_get_offset(NC *ncp, NC_var *varp, const MPI_Offset starts[], /* [varp->ndims] */ const MPI_Offset counts[], /* [varp->ndims] */ const MPI_Offset strides[], /* [varp->ndims] */ MPI_Offset *offset_ptr) /* return file offset */ { /* returns the starting file offset when this variable is get/put with starts[] */ MPI_Offset offset, *end_off=NULL; int status, i, ndims; offset = varp->begin; /* beginning file offset of this variable */ ndims = varp->ndims; /* number of dimensions of this variable */ if (counts != NULL) end_off = (MPI_Offset*) NCI_Malloc(ndims * sizeof(MPI_Offset)); if (counts != NULL && strides != NULL) { for (i=0; i<ndims; i++) end_off[i] = starts[i] + counts[i] * strides[i] - 1; } else if (counts != NULL) { /* strides == NULL */ for (i=0; i<ndims; i++) end_off[i] = starts[i] + counts[i] - 1; } else { /* when counts == NULL strides is of no use */ end_off = (MPI_Offset*) starts; } status = NCcoordck(ncp, varp, end_off); /* validate end_off[] */ if (status != NC_NOERR) { #ifdef CDEBUG printf("ncmpii_get_offset(): NCcoordck() fails\n"); #endif return status; } if (ndims > 0) { if (IS_RECVAR(varp)) /* no need to check recsize here: if MPI_Offset is only 32 bits we will have had problems long before here */ offset += end_off[0] * ncp->recsize; else offset += end_off[ndims-1] * varp->xsz; if (ndims > 1) { if (IS_RECVAR(varp)) offset += end_off[ndims - 1] * varp->xsz; else offset += end_off[0] * varp->dsizes[1] * varp->xsz; for (i=1; i<ndims-1; i++) offset += end_off[i] * varp->dsizes[i+1] * varp->xsz; } } if (counts != NULL && end_off != NULL) NCI_Free(end_off); *offset_ptr = offset; return NC_NOERR; }
/*----< ncmpii_vara_create_filetype() >--------------------------------------*/ static int ncmpii_vara_create_filetype(NC *ncp, NC_var *varp, const MPI_Offset *start, const MPI_Offset *count, int rw_flag, MPI_Offset *offset_ptr, MPI_Datatype *filetype_ptr) { int dim, status; MPI_Offset offset, nelems=1; MPI_Datatype filetype; offset = varp->begin; filetype = MPI_BYTE; /* New coordinate/edge check to fix NC_EINVALCOORDS bug */ status = NCedgeck(ncp, varp, start, count); if (status != NC_NOERR || (rw_flag == READ_REQ && IS_RECVAR(varp) && *start + *count > NC_get_numrecs(ncp))) { status = NCcoordck(ncp, varp, start); if (status != NC_NOERR) return status; else return NC_EEDGE; } /* check if the request is contiguous in file if yes, there is no need to create a filetype */ if (ncmpii_is_request_contiguous(varp, start, count)) { status = ncmpii_get_offset(ncp, varp, start, NULL, NULL, &offset); *offset_ptr = offset; *filetype_ptr = filetype; return status; } for (dim=0; dim<varp->ndims; dim++) nelems *= count[dim]; /* filetype is defined only when varp is not a scalar and the number of requested elemenst > 0 (varp->ndims == 0 meaning this is a scalar variable) Otherwise, keep filetype MPI_BYTE */ if (varp->ndims > 0 && nelems > 0) { int i, ndims, blklens[3], tag=0; int *shape=NULL, *subcount=NULL, *substart=NULL; /* all in bytes */ MPI_Offset *shape64=NULL, *subcount64=NULL, *substart64=NULL; MPI_Offset size, disps[3]; MPI_Datatype rectype, types[3], type1; ndims = varp->ndims; shape = (int*) NCI_Malloc(3 * ndims * sizeof(int)); subcount = shape + ndims; substart = subcount + ndims; /* here, request size has been checked and it must > 0 */ if (IS_RECVAR(varp)) { subcount[0] = count[0]; substart[0] = 0; shape[0] = subcount[0]; if (ncp->recsize <= varp->len) { /* the only record variable */ if (varp->ndims == 1) { shape[0] *= varp->xsz; subcount[0] *= varp->xsz; } else { for (dim = 1; dim < ndims-1; dim++) { shape[dim] = varp->shape[dim]; subcount[dim] = count[dim]; substart[dim] = start[dim]; } shape[dim] = varp->xsz * varp->shape[dim]; subcount[dim] = varp->xsz * count[dim]; substart[dim] = varp->xsz * start[dim]; } offset += start[0] * ncp->recsize; MPI_Type_create_subarray(ndims, shape, subcount, substart, MPI_ORDER_C, MPI_BYTE, &filetype); MPI_Type_commit(&filetype); } else { check_recsize_too_big(ncp); /* more than one record variables */ offset += start[0] * ncp->recsize; if (varp->ndims == 1) { #if (MPI_VERSION < 2) MPI_Type_hvector(subcount[0], varp->xsz, ncp->recsize, MPI_BYTE, &filetype); #else MPI_Type_create_hvector(subcount[0], varp->xsz, ncp->recsize, MPI_BYTE, &filetype); #endif MPI_Type_commit(&filetype); } else { for (dim = 1; dim < ndims-1; dim++) { shape[dim] = varp->shape[dim]; subcount[dim] = count[dim]; substart[dim] = start[dim]; } shape[dim] = varp->xsz * varp->shape[dim]; subcount[dim] = varp->xsz * count[dim]; substart[dim] = varp->xsz * start[dim]; MPI_Type_create_subarray(ndims-1, shape+1, subcount+1, substart+1, MPI_ORDER_C, MPI_BYTE, &rectype); MPI_Type_commit(&rectype); #if (MPI_VERSION < 2) MPI_Type_hvector(subcount[0], 1, ncp->recsize, rectype, &filetype); #else MPI_Type_create_hvector(subcount[0], 1, ncp->recsize, rectype, &filetype); #endif MPI_Type_commit(&filetype); MPI_Type_free(&rectype); } } } else { /* non record variable */ tag = 0; for (dim=0; dim< ndims-1; dim++) { if (varp->shape[dim] > 2147483647) { /* if shape > 2^31-1 */ tag = 1; break; } } if ((varp->shape[dim]*varp->xsz) > 2147483647) tag = 1; if (tag == 0) { for (dim = 0; dim < ndims-1; dim++ ) { shape[dim] = varp->shape[dim]; subcount[dim] = count[dim]; substart[dim] = start[dim]; } shape[dim] = varp->xsz * varp->shape[dim]; subcount[dim] = varp->xsz * count[dim]; substart[dim] = varp->xsz * start[dim]; MPI_Type_create_subarray(ndims, shape, subcount, substart, MPI_ORDER_C, MPI_BYTE, &filetype); MPI_Type_commit(&filetype); } else { shape64 = (MPI_Offset*) NCI_Malloc(3 * ndims * sizeof(MPI_Offset)); subcount64 = shape64 + ndims; substart64 = subcount64 + ndims; if (ndims == 1) { // for 64-bit support, added July 23, 2008 shape64[0] = varp->shape[0]; subcount64[0] = count[0]; substart64[0] = start[0]; offset += start[0]*varp->xsz; MPI_Type_contiguous(subcount64[0]*varp->xsz, MPI_BYTE, &type1); MPI_Type_commit(&type1); #if (MPI_VERSION < 2) MPI_Type_hvector(subcount64[0], varp->xsz, shape64[0]*varp->xsz, MPI_BYTE, &filetype); #else MPI_Type_create_hvector(1, 1, shape64[0]*varp->xsz, type1, &filetype); #endif MPI_Type_commit(&filetype); MPI_Type_free(&type1); } else { for (dim = 0; dim < ndims-1; dim++ ) { shape64[dim] = varp->shape[dim]; subcount64[dim] = count[dim]; substart64[dim] = start[dim]; } shape64[dim] = varp->xsz * varp->shape[dim]; subcount64[dim] = varp->xsz * count[dim]; substart64[dim] = varp->xsz * start[dim]; MPI_Type_hvector(subcount64[dim-1], subcount64[dim], varp->xsz * varp->shape[dim], MPI_BYTE, &type1); MPI_Type_commit(&type1); size = shape[dim]; for (i=dim-2; i>=0; i--) { size *= shape[i+1]; MPI_Type_hvector(subcount64[i], 1, size, type1, &filetype); MPI_Type_commit(&filetype); MPI_Type_free(&type1); type1 = filetype; } disps[1] = substart64[dim]; size = 1; for (i=dim-1; i>=0; i--) { size *= shape64[i+1]; disps[1] += size*substart64[i]; } disps[2] = 1; for (i=0; i<ndims; i++) disps[2] *= shape64[i]; disps[0] = 0; blklens[0] = blklens[1] = blklens[2] = 1; types[0] = MPI_LB; types[1] = type1; types[2] = MPI_UB; MPI_Type_struct(3, blklens, (MPI_Aint*) disps, types, &filetype); MPI_Type_free(&type1); } NCI_Free(shape64); } } NCI_Free(shape); } *offset_ptr = offset; *filetype_ptr = filetype; return NC_NOERR; }
/* * Recompute the shapes of all variables * Sets ncp->begin_var to start of first variable. * Sets ncp->begin_rec to start of first record variable. * Returns -1 on error. The only possible error is a reference * to a non existent dimension, which could occur for a corrupted * netcdf file. */ static int NC_computeshapes(NC *ncp) { NC_var **vpp = (NC_var **)ncp->vars.value; NC_var *const *const end = &vpp[ncp->vars.nelems]; NC_var *first_var = NULL; /* first "non-record" var */ NC_var *first_rec = NULL; /* first "record" var */ int status; ncp->begin_var = (off_t) ncp->xsz; ncp->begin_rec = (off_t) ncp->xsz; ncp->recsize = 0; if(ncp->vars.nelems == 0) return(0); for( /*NADA*/; vpp < end; vpp++) { status = NC_var_shape(*vpp, &ncp->dims); if(status != ENOERR) return(status); if(IS_RECVAR(*vpp)) { if(first_rec == NULL) first_rec = *vpp; ncp->recsize += (*vpp)->len; } else { if(first_var == NULL) first_var = *vpp; /* * Overwritten each time thru. * Usually overwritten in first_rec != NULL clause below. */ ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; } } if(first_rec != NULL) { assert(ncp->begin_rec <= first_rec->begin); ncp->begin_rec = first_rec->begin; /* * for special case of exactly one record variable, pack value */ if(ncp->recsize == first_rec->len) ncp->recsize = *first_rec->dsizes * first_rec->xsz; } if(first_var != NULL) { ncp->begin_var = first_var->begin; } else { ncp->begin_var = ncp->begin_rec; } assert(ncp->begin_var > 0); assert(ncp->xsz <= (size_t)ncp->begin_var); assert(ncp->begin_rec > 0); assert(ncp->begin_var <= ncp->begin_rec); return(ENOERR); }
/* * 'compile' the shape and len of a variable * Formerly NC_var_shape(var, dims) */ int NC_var_shape(NC_var *varp, const NC_dimarray *dims) { size_t *shp, *dsp, *op; int *ip; const NC_dim *dimp; size_t product = 1; varp->xsz = ncx_szof(varp->type); if(varp->ndims == 0) { goto out; } /* * use the user supplied dimension indices * to determine the shape */ for(ip = varp->dimids, op = varp->shape ; ip < &varp->dimids[varp->ndims]; ip++, op++) { if(*ip < 0 || (size_t) (*ip) >= ((dims != NULL) ? dims->nelems : 1) ) return NC_EBADDIM; dimp = elem_NC_dimarray(dims, (size_t)*ip); *op = dimp->size; if(*op == NC_UNLIMITED && ip != varp->dimids) return NC_EUNLIMPOS; } /* * Compute the dsizes */ /* ndims is > 0 here */ for(shp = varp->shape + varp->ndims -1, dsp = varp->dsizes + varp->ndims -1; shp >= varp->shape; shp--, dsp--) { if(!(shp == varp->shape && IS_RECVAR(varp))) product *= *shp; *dsp = product; } out : varp->len = product * varp->xsz; switch(varp->type) { case NC_BYTE : case NC_CHAR : case NC_SHORT : if( varp->len%4 != 0 ) { varp->len += 4 - varp->len%4; /* round up */ /* *dsp += 4 - *dsp%4; */ } break; default: /* already aligned */ break; } #if 0 arrayp("\tshape", varp->ndims, varp->shape); arrayp("\tdsizes", varp->ndims, varp->dsizes); #endif return NC_NOERR; }
/* * Compute each variable's 'begin' offset, * update 'begin_rec' as well. */ static int NC_begins(NC *ncp, size_t h_minfree, size_t v_align, size_t v_minfree, size_t r_align) { size_t ii; int sizeof_off_t; off_t index = 0; NC_var **vpp; NC_var *last = NULL; if(v_align == NC_ALIGN_CHUNK) v_align = ncp->chunk; if(r_align == NC_ALIGN_CHUNK) r_align = ncp->chunk; if (fIsSet(ncp->flags, NC_64BIT_OFFSET)) { sizeof_off_t = 8; } else { sizeof_off_t = 4; } ncp->xsz = ncx_len_NC(ncp,sizeof_off_t); if(ncp->vars.nelems == 0) return NC_NOERR; /* only (re)calculate begin_var if there is not sufficient space in header or start of non-record variables is not aligned as requested by valign */ if (ncp->begin_var < ncp->xsz + h_minfree || ncp->begin_var != D_RNDUP(ncp->begin_var, v_align) ) { index = (off_t) ncp->xsz; ncp->begin_var = D_RNDUP(index, v_align); if(ncp->begin_var < index + h_minfree) { ncp->begin_var = D_RNDUP(index + (off_t)h_minfree, v_align); } } index = ncp->begin_var; /* loop thru vars, first pass is for the 'non-record' vars */ vpp = ncp->vars.value; for(ii = 0; ii < ncp->vars.nelems ; ii++, vpp++) { if( IS_RECVAR(*vpp) ) { /* skip record variables on this pass */ continue; } #if 0 fprintf(stderr, " VAR %d %s: %ld\n", ii, (*vpp)->name->cp, (long)index); #endif if( sizeof_off_t == 4 && (index > X_OFF_MAX || index < 0) ) { return NC_EVARSIZE; } (*vpp)->begin = index; index += (*vpp)->len; } /* only (re)calculate begin_rec if there is not sufficient space at end of non-record variables or if start of record variables is not aligned as requested by r_align */ if (ncp->begin_rec < index + v_minfree || ncp->begin_rec != D_RNDUP(ncp->begin_rec, r_align) ) { ncp->begin_rec = D_RNDUP(index, r_align); if(ncp->begin_rec < index + v_minfree) { ncp->begin_rec = D_RNDUP(index + (off_t)v_minfree, r_align); } } index = ncp->begin_rec; ncp->recsize = 0; /* loop thru vars, second pass is for the 'record' vars */ vpp = (NC_var **)ncp->vars.value; for(ii = 0; ii < ncp->vars.nelems; ii++, vpp++) { if( !IS_RECVAR(*vpp) ) { /* skip non-record variables on this pass */ continue; } #if 0 fprintf(stderr, " REC %d %s: %ld\n", ii, (*vpp)->name->cp, (long)index); #endif if( sizeof_off_t == 4 && (index > X_OFF_MAX || index < 0) ) { return NC_EVARSIZE; } (*vpp)->begin = index; index += (*vpp)->len; /* check if record size must fit in 32-bits */ #if SIZEOF_OFF_T == SIZEOF_SIZE_T && SIZEOF_SIZE_T == 4 if( ncp->recsize > X_UINT_MAX - (*vpp)->len ) { return NC_EVARSIZE; } #endif ncp->recsize += (*vpp)->len; last = (*vpp); } /* * for special case of exactly one record variable, pack value */ if(last != NULL && ncp->recsize == last->len) ncp->recsize = *last->dsizes * last->xsz; if(NC_IsNew(ncp)) NC_set_numrecs(ncp, 0); return NC_NOERR; }
/* * Given a valid ncp, return NC_EVARSIZE if any variable has a bad len * (product of non-rec dim sizes too large), else return NC_NOERR. */ static int NC_check_vlens(NC *ncp) { NC_var **vpp; /* maximum permitted variable size (or size of one record's worth of a record variable) in bytes. This is different for format 1 and format 2. */ size_t vlen_max; size_t ii; size_t large_vars_count; size_t rec_vars_count; int last = 0; if(ncp->vars.nelems == 0) return NC_NOERR; if ((ncp->flags & NC_64BIT_OFFSET) && sizeof(off_t) > 4) { /* CDF2 format and LFS */ vlen_max = X_UINT_MAX - 3; /* "- 3" handles rounded-up size */ } else { /* CDF1 format */ vlen_max = X_INT_MAX - 3; } /* Loop through vars, first pass is for non-record variables. */ large_vars_count = 0; rec_vars_count = 0; vpp = ncp->vars.value; for (ii = 0; ii < ncp->vars.nelems; ii++, vpp++) { if( !IS_RECVAR(*vpp) ) { last = 0; if( NC_check_vlen(*vpp, vlen_max) == 0 ) { large_vars_count++; last = 1; } } else { rec_vars_count++; } } /* OK if last non-record variable size too large, since not used to compute an offset */ if( large_vars_count > 1) { /* only one "too-large" variable allowed */ return NC_EVARSIZE; } /* and it has to be the last one */ if( large_vars_count == 1 && last == 0) { return NC_EVARSIZE; } if( rec_vars_count > 0 ) { /* and if it's the last one, there can't be any record variables */ if( large_vars_count == 1 && last == 1) { return NC_EVARSIZE; } /* Loop through vars, second pass is for record variables. */ large_vars_count = 0; vpp = ncp->vars.value; for (ii = 0; ii < ncp->vars.nelems; ii++, vpp++) { if( IS_RECVAR(*vpp) ) { last = 0; if( NC_check_vlen(*vpp, vlen_max) == 0 ) { large_vars_count++; last = 1; } } } /* OK if last record variable size too large, since not used to compute an offset */ if( large_vars_count > 1) { /* only one "too-large" variable allowed */ return NC_EVARSIZE; } /* and it has to be the last one */ if( large_vars_count == 1 && last == 0) { return NC_EVARSIZE; } } return NC_NOERR; }