/* Given the arguments to vara
construct a corresponding projection
with any pseudo dimensions removed
*/
NCerror
buildvaraprojection3(Getvara* getvar,
const size_t* startp, const size_t* countp, const ptrdiff_t* stridep,
DCEprojection** projectionp)
{
int i,j;
NCerror ncstat = NC_NOERR;
CDFnode* var = getvar->target;
DCEprojection* projection = NULL;
NClist* path = nclistnew();
NClist* segments = NULL;
int dimindex;
ncstat = dapvar2projection(var,&projection);
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection: %s\n",dumpprojection(projection));
#endif
/* We need to assign the start/count/stride info to each segment;
declsize will have been set
*/
segments = projection->var->segments;
dimindex = 0;
for(i=0;i<nclistlength(segments);i++) {
DCEsegment* segment = (DCEsegment*)nclistget(segments,i);
for(j=0;j<segment->rank;j++) {
DCEslice* slice = &segment->slices[j];
/* make each slice represent the corresponding
start/count/stride */
slice->first = startp[dimindex+j];
slice->stride = stridep[dimindex+j];
slice->count = countp[dimindex+j];
slice->length = slice->count * slice->stride;
if(slice->length > slice->declsize)
slice->length = slice->declsize;
slice->stop = (slice->first + slice->length);
if(slice->stop > slice->declsize)
slice->stop = slice->declsize;
}
dimindex += segment->rank;
}
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection.final: %s\n",dumpprojection(projection));
#endif
#ifdef IGNORE
removepseudodims3(projection);
#endif
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection3: projection=%s\n",
dumpprojection(projection));
#endif
if(projectionp) *projectionp = projection;
nclistfree(path);
if(ncstat) dcefree((DCEnode*)projection);
return ncstat;
}
/* Compute the set of prefetched data.
Notes:
1. All prefetches are whole variable fetches.
2. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i;
NCFLAGS flags;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else {
/* pull in those variables previously marked as prefetchable */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
/* Most of the important testing was already done */
if(!var->basenode->prefetchable)
continue;
/* Do not attempt to prefetch any variables in the
nc_open url's projection list
*/
if(nclistcontains(nccomm->cdf.projectedvars,(void*)var))
continue;
/* Should be prefetchable */
nclistpush(vars,(void*)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
The exception is if we are prefetching everything.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(void*)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
flags = NCF_PREFETCH;
if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL;
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
/* Given the arguments to vara
construct a corresponding projection
with any pseudo dimensions removed
*/
NCerror
buildvaraprojection(CDFnode* var,
const size_t* startp, const size_t* countp, const ptrdiff_t* stridep,
DCEprojection** projectionp)
{
int i,j;
NCerror ncstat = NC_NOERR;
DCEprojection* projection = NULL;
NClist* path = nclistnew();
NClist* segments = NULL;
int dimindex;
/* Build a skeleton projection that has 1 segment for
every cdfnode from root to the variable of interest.
Each segment has the slices from its corresponding node
in the path, including pseudo-dims
*/
ncstat = dapvar2projection(var,&projection);
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection: skeleton: %s\n",dumpprojection(projection));
#endif
/* Now, modify the projection to reflect the corresponding
start/count/stride from the nc_get_vara arguments.
*/
segments = projection->var->segments;
dimindex = 0;
for(i=0;i<nclistlength(segments);i++) {
DCEsegment* segment = (DCEsegment*)nclistget(segments,i);
for(j=0;j<segment->rank;j++) {
size_t count = 0;
DCEslice* slice = &segment->slices[j];
/* make each slice represent the corresponding
start/count/stride */
slice->first = startp[dimindex+j];
slice->stride = stridep[dimindex+j];
count = countp[dimindex+j];
slice->count = count;
slice->length = count * slice->stride;
slice->last = (slice->first + slice->length) - 1;
if(slice->last >= slice->declsize) {
slice->last = slice->declsize - 1;
/* reverse compute the new length */
slice->length = (slice->last - slice->first) + 1;
}
}
dimindex += segment->rank;
}
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection.final: %s\n",dumpprojection(projection));
#endif
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection3: final: projection=%s\n",
dumpprojection(projection));
#endif
if(projectionp) *projectionp = projection;
nclistfree(path);
if(ncstat) dcefree((DCEnode*)projection);
return ncstat;
}
/* Compute the set of prefetched data.
Notes:
1. Even if caching is off, we will
still prefetch the small variables.
2. All prefetches are whole variable fetches.
3. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
int isnc4 = FLAGSET(nccomm->controls,NCF_NC4);
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
if(!isnc4) {
/* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
}
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimset0);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j);
nelems *= dim->dim.declsize;
}
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems);
}
if(nelems <= nccomm->cdf.smallsizelimit) {
nclistpush(vars,(ncelem)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(ncelem)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
