/*
A variable is prefetchable if
1. it is atomic
2. it's size is sufficiently small
3. it is not contained in sequence or a dimensioned structure.
*/
NCerror
markprefetch(NCDAPCOMMON* nccomm)
{
int i,j;
NClist* allvars = nccomm->cdf.fullddsroot->tree->varnodes;
assert(allvars != NULL);
/* mark those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems;
/* If var is not atomic, then it is not prefetchable */
if(var->nctype != NC_Atomic)
continue;
/* if var is under a sequence, then never prefetch */
if(dapinsequence(var))
continue;
/* Compute the # of elements in the variable */
for(nelems=1,j=0;j<nclistlength(var->array.dimsettrans);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimsettrans,j);
nelems *= dim->dim.declsize;
}
if(nelems <= nccomm->cdf.smallsizelimit
&& FLAGSET(nccomm->controls,NCF_PREFETCH)) {
var->prefetchable = 1;
if(SHOWFETCH)
{
extern char* ocfqn(OCddsnode);
char *tmp = ocfqn(var->ocnode);
nclog(NCLOGDBG,"prefetchable: %s=%lu",
tmp,(unsigned long)nelems);
free(tmp);
}
}
}
return NC_NOERR;
}
/* Compute the set of prefetched data;
note that even if caching is off, we will
still prefetch the small variables.
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
NCconstraint* constraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
NCconstraint* newconstraint = NULL;
/* Check if we can do constraints */
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) { /*cannot constrain*/
/* If we cannot constrain, then pull in everything */
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
/* If var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimensions);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimensions,j);
nelems *= dim->dim.declsize;
}
if(nelems <= nccomm->cdf.smallsizelimit)
nclistpush(vars,(ncelem)var);
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
newconstraint = createncconstraint();
/* Construct the projections for this set of vars */
/* Initially, the constraints are same as the merged constraints */
newconstraint->projections = clonencprojections(constraint->projections);
restrictprojection34(vars,newconstraint->projections);
/* similar for selections */
newconstraint->selections = clonencselections(constraint->selections);
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,1);
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
if(FLAGSET(nccomm->controls,NCF_SHOWFETCH)) {
oc_log(OCLOGNOTE,"prefetch.");
}
#ifdef DEBUG
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
ncbytescat(buf,makesimplepathstring3(var));
}
ncbytescat(buf,"\n");
oc_log(OCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
#endif
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
done:
nclistfree(vars);
freencconstraint(newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
/* Compute the set of prefetched data.
Notes:
1. Even if caching is off, we will
still prefetch the small variables.
2. All prefetches are whole variable fetches.
3. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
int isnc4 = FLAGSET(nccomm->controls,NCF_NC4);
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
if(!isnc4) {
/* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
}
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimset0);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j);
nelems *= dim->dim.declsize;
}
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems);
}
if(nelems <= nccomm->cdf.smallsizelimit) {
nclistpush(vars,(ncelem)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(ncelem)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
