static DCEprojection*
projectify(CDFnode* field, DCEprojection* container)
{
DCEprojection* proj = (DCEprojection*)dcecreate(CES_PROJECT);
DCEvar* var = (DCEvar*)dcecreate(CES_VAR);
DCEsegment* seg = (DCEsegment*)dcecreate(CES_SEGMENT);
proj->discrim = CES_VAR;
proj->var = var;
var->annotation = (void*)field;
/* Dup the segment list */
var->segments = dceclonelist(container->var->segments);
seg->rank = 0;
nclistpush(var->segments,(ncelem)seg);
return proj;
}
DCEnode*
dceclone(DCEnode* node)
{
DCEnode* result = NULL;
result = (DCEnode*)dcecreate(node->sort);
if(result == NULL) goto done;
switch (node->sort) {
case CES_SLICE: {
DCEslice* clone = (DCEslice*)result;
DCEslice* orig = (DCEslice*)node;
*clone = *orig;
} break;
case CES_SEGMENT: {
DCEsegment* clone = (DCEsegment*)result;
DCEsegment* orig = (DCEsegment*)node;
*clone = *orig;
clone->name = nulldup(orig->name);
if(orig->rank > 0)
memcpy(clone->slices,orig->slices,orig->rank*sizeof(DCEslice));
} break;
case CES_VAR: {
DCEvar* clone = (DCEvar*)result;
DCEvar* orig = (DCEvar*)node;
*clone = *orig;
clone->segments = dceclonelist(clone->segments);
} break;
case CES_FCN: {
DCEfcn* clone = (DCEfcn*)result;
DCEfcn* orig = (DCEfcn*)node;
*clone = *orig;
clone->name = nulldup(orig->name);
clone->args = dceclonelist(orig->args);
} break;
case CES_CONST: {
DCEconstant* clone = (DCEconstant*)result;
DCEconstant* orig = (DCEconstant*)node;
*clone = *orig;
if(clone->discrim == CES_STR)
clone->text = nulldup(clone->text);
} break;
case CES_VALUE: {
DCEvalue* clone = (DCEvalue*)result;
DCEvalue* orig = (DCEvalue*)node;
*clone = *orig;
switch (clone->discrim) {
case CES_CONST:
clone->constant = (DCEconstant*)dceclone((DCEnode*)orig->constant); break;
case CES_VAR:
clone->var = (DCEvar*)dceclone((DCEnode*)orig->var); break;
case CES_FCN:
clone->fcn = (DCEfcn*)dceclone((DCEnode*)orig->fcn); break;
default: assert(0);
}
} break;
case CES_PROJECT: {
DCEprojection* clone = (DCEprojection*)result;
DCEprojection* orig = (DCEprojection*)node;
*clone = *orig;
switch (orig->discrim) {
case CES_VAR:
clone->var = (DCEvar*)dceclone((DCEnode*)orig->var); break;
case CES_FCN:
clone->fcn = (DCEfcn*)dceclone((DCEnode*)orig->fcn); break;
default: assert(0);
}
} break;
case CES_SELECT: {
DCEselection* clone = (DCEselection*)result;
DCEselection* orig = (DCEselection*)node;
*clone = *orig;
clone->lhs = (DCEvalue*)dceclone((DCEnode*)orig->lhs);
clone->rhs = dceclonelist(orig->rhs);
} break;
case CES_CONSTRAINT: {
DCEconstraint* clone = (DCEconstraint*)result;
DCEconstraint* orig = (DCEconstraint*)node;
*clone = *orig;
clone->projections = dceclonelist(orig->projections);
clone->selections = dceclonelist(orig->selections);
} break;
default:
assert(0);
}
done:
return result;
}
/* Compute the set of prefetched data.
Notes:
1. All prefetches are whole variable fetches.
2. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i;
NCFLAGS flags;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else {
/* pull in those variables previously marked as prefetchable */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
/* Most of the important testing was already done */
if(!var->basenode->prefetchable)
continue;
/* Do not attempt to prefetch any variables in the
nc_open url's projection list
*/
if(nclistcontains(nccomm->cdf.projectedvars,(void*)var))
continue;
/* Should be prefetchable */
nclistpush(vars,(void*)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
The exception is if we are prefetching everything.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(void*)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
flags = NCF_PREFETCH;
if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL;
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
NCerror
nc3d_getvarx(int ncid, int varid,
const size_t *startp,
const size_t *countp,
const ptrdiff_t* stridep,
void *data,
nc_type dsttype0)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
int i;
NC* drno;
NC* substrate;
NCDAPCOMMON* dapcomm;
CDFnode* cdfvar = NULL; /* cdf node mapping to var*/
NClist* varnodes;
nc_type dsttype;
Getvara* varainfo = NULL;
CDFnode* xtarget = NULL; /* target in DATADDS */
CDFnode* target = NULL; /* target in constrained DDS */
DCEprojection* varaprojection = NULL;
NCcachenode* cachenode = NULL;
size_t localcount[NC_MAX_VAR_DIMS];
NClist* ncdimsall;
size_t ncrank;
NClist* vars = NULL;
DCEconstraint* fetchconstraint = NULL;
DCEprojection* fetchprojection = NULL;
DCEprojection* walkprojection = NULL;
int state;
#define FETCHWHOLE 1 /* fetch whole data set */
#define FETCHVAR 2 /* fetch whole variable */
#define FETCHPART 4 /* fetch constrained variable */
#define CACHED 8 /* whole variable is already in the cache */
ncstat = NC_check_id(ncid, (NC**)&drno);
if(ncstat != NC_NOERR) goto fail;
dapcomm = (NCDAPCOMMON*)drno->dispatchdata;
ncstat = NC_check_id(drno->substrate, (NC**)&substrate);
if(ncstat != NC_NOERR) goto fail;
/* Locate var node via varid */
varnodes = dapcomm->cdf.ddsroot->tree->varnodes;
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(varnodes,i);
if(node->array.basevar == NULL
&& node->nctype == NC_Atomic
&& node->ncid == varid) {
cdfvar = node;
break;
}
}
ASSERT((cdfvar != NULL));
/* If the variable is prefetchable, then now
is the time to do a lazy prefetch */
if(FLAGSET(dapcomm->controls,NCF_PREFETCH)
&& !FLAGSET(dapcomm->controls,NCF_PREFETCH_EAGER)) {
if(dapcomm->cdf.cache != NULL && dapcomm->cdf.cache->prefetch == NULL) {
ncstat = prefetchdata3(dapcomm);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
}
}
/* Get the dimension info */
ncdimsall = cdfvar->array.dimsetall;
ncrank = nclistlength(ncdimsall);
#ifdef DEBUG
{
int i;
fprintf(stderr,"getvarx: %s",cdfvar->ncfullname);
for(i=0;i<ncrank;i++)
fprintf(stderr,"(%ld:%ld:%ld)",
(long)startp[i],
(long)countp[i],
(long)stridep[i]
);
fprintf(stderr,"\n");
}
#endif
/* Fill in missing arguments */
if(startp == NULL)
startp = nc_sizevector0;
if(countp == NULL) {
/* Accumulate the dimension sizes */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdimsall,i);
localcount[i] = dim->dim.declsize;
}
countp = localcount;
}
if(stridep == NULL)
stridep = nc_ptrdiffvector1;
/* Validate the dimension sizes */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdimsall,i);
if(startp[i] > dim->dim.declsize
|| startp[i]+countp[i] > dim->dim.declsize) {
ncstat = NC_EINVALCOORDS;
goto fail;
}
}
#ifdef DEBUG
{
NClist* dims = cdfvar->array.dimsetall;
fprintf(stderr,"getvarx: %s",cdfvar->ncfullname);
if(nclistlength(dims) > 0) {int i;
for(i=0;i<nclistlength(dims);i++)
fprintf(stderr,"(%lu:%lu:%lu)",(unsigned long)startp[i],(unsigned long)countp[i],(unsigned long)stridep[i]);
fprintf(stderr," -> ");
for(i=0;i<nclistlength(dims);i++)
if(stridep[i]==1)
fprintf(stderr,"[%lu:%lu]",(unsigned long)startp[i],(unsigned long)((startp[i]+countp[i])-1));
else {
unsigned long iend = (stridep[i] * countp[i]);
iend = (iend + startp[i]);
iend = (iend - 1);
fprintf(stderr,"[%lu:%lu:%lu]",
(unsigned long)startp[i],(unsigned long)stridep[i],iend);
}
}
fprintf(stderr,"\n");
}
#endif
dsttype = (dsttype0);
/* Default to using the inquiry type for this var*/
if(dsttype == NC_NAT) dsttype = cdfvar->externaltype;
/* Validate any implied type conversion*/
if(cdfvar->etype != dsttype && dsttype == NC_CHAR) {
/* The only disallowed conversion is to/from char and non-byte
numeric types*/
switch (cdfvar->etype) {
case NC_STRING: case NC_URL:
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
break;
default:
return THROW(NC_ECHAR);
}
}
ncstat = makegetvar34(dapcomm,cdfvar,data,dsttype,&varainfo);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Compile the start/stop/stride info into a projection */
ncstat = buildvaraprojection3(varainfo->target,
startp,countp,stridep,
&varaprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
fetchprojection = NULL;
walkprojection = NULL;
/* Create walkprojection as the merge of the url projections
and the vara projection; may change in FETCHPART case below*/
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&walkprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
#ifdef DEBUG
fprintf(stderr,"getvarx: walkprojection: |%s|\n",dumpprojection(walkprojection));
#endif
/* define the var list of interest */
vars = nclistnew();
nclistpush(vars,(void*)varainfo->target);
state = 0;
if(iscached(dapcomm,cdfvar,&cachenode)) { /* ignores non-whole variable cache entries */
state = CACHED;
ASSERT((cachenode != NULL));
#ifdef DEBUG
fprintf(stderr,"var is in cache\n");
#endif
/* If it is cached, then it is a whole variable but may still
need to apply constraints during the walk */
ASSERT(cachenode->wholevariable); /* by construction */
} else if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
state = FETCHWHOLE;
} else {/* load using constraints */
if(FLAGSET(dapcomm->controls,NCF_WHOLEVAR))
state = FETCHVAR;
else
state = FETCHPART;
}
ASSERT(state != 0);
switch (state) {
case FETCHWHOLE: {
/* buildcachenode3 will create a new cachenode and
will also fetch the whole corresponding datadds.
*/
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* Use no projections or selections */
fetchconstraint->projections = nclistnew();
fetchconstraint->selections = nclistnew();
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHWHOLE: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
case CACHED: {
} break;
case FETCHVAR: { /* Fetch a complete single variable */
/* Create fetch projection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
/* elide any sequence and string dimensions (dap servers do not allow such). */
ncstat = removepseudodims(fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Convert to a whole variable projection */
dcemakewholeprojection(fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHVAR: fetchprojection: |%s|\n",dumpprojection(fetchprojection));
#endif
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* merged constraint just uses the url constraint selection */
fetchconstraint->selections = dceclonelist(dapcomm->oc.dapconstraint->selections);
/* and the created fetch projection */
fetchconstraint->projections = nclistnew();
nclistpush(fetchconstraint->projections,(void*)fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHVAR: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
/* buildcachenode3 will create a new cachenode and
will also fetch the corresponding datadds.
*/
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
case FETCHPART: {
/* Create fetch projection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
/* elide any sequence and string dimensions (dap servers do not allow such). */
ncstat = removepseudodims(fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Shift the varaprojection for simple walk */
dcefree((DCEnode*)walkprojection) ; /* reclaim any existing walkprojection */
walkprojection = (DCEprojection*)dceclone((DCEnode*)varaprojection);
dapshiftprojection(walkprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHPART: fetchprojection: |%s|\n",dumpprojection(fetchprojection));
#endif
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* merged constraint just uses the url constraint selection */
fetchconstraint->selections = dceclonelist(dapcomm->oc.dapconstraint->selections);
/* and the created fetch projection */
fetchconstraint->projections = nclistnew();
nclistpush(fetchconstraint->projections,(void*)fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHPART: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
/* buildcachenode3 will create a new cachenode and
will also fetch the corresponding datadds.
*/
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
default: PANIC1("unknown fetch state: %d\n",state);
}
ASSERT(cachenode != NULL);
#ifdef DEBUG
fprintf(stderr,"cache.datadds=%s\n",dumptree(cachenode->datadds));
#endif
/* attach DATADDS to (constrained) DDS */
unattach34(dapcomm->cdf.ddsroot);
ncstat = attachsubset34(cachenode->datadds,dapcomm->cdf.ddsroot);
if(ncstat) goto fail;
/* Fix up varainfo to use the cache */
varainfo->cache = cachenode;
cachenode = NULL;
varainfo->varaprojection = walkprojection;
walkprojection = NULL;
/* Get the var correlate from the datadds */
target = varainfo->target;
xtarget = target->attachment;
if(xtarget == NULL)
{THROWCHK(ncstat=NC_ENODATA); goto fail;}
/* Switch to datadds tree space*/
varainfo->target = xtarget;
save = (DCEnode*)varaprojection;
ncstat = moveto(dapcomm,varainfo,varainfo->cache->datadds,data);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
nclistfree(vars);
dcefree((DCEnode*)varaprojection);
dcefree((DCEnode*)fetchconstraint);
freegetvara(varainfo);
fail:
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
/* Compute the set of prefetched data.
Notes:
1. Even if caching is off, we will
still prefetch the small variables.
2. All prefetches are whole variable fetches.
3. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
int isnc4 = FLAGSET(nccomm->controls,NCF_NC4);
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
if(!isnc4) {
/* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
}
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimset0);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j);
nelems *= dim->dim.declsize;
}
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems);
}
if(nelems <= nccomm->cdf.smallsizelimit) {
nclistpush(vars,(ncelem)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(ncelem)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
