void
freenccachenode(NCDAPCOMMON* nccomm, NCcachenode* node)
{
if(node == NULL) return;
#ifdef DEBUG
fprintf(stderr,"freecachenode: %s\n",
dumpcachenode(node));
#endif
oc_data_free(nccomm->oc.conn,node->content);
dcefree((DCEnode*)node->constraint);
freecdfroot(node->datadds);
nclistfree(node->vars);
nullfree(node);
}
void
dcefree(DCEnode* node)
{
if(node == NULL) return;
switch (node->sort) {
case CES_VAR: {
DCEvar* target = (DCEvar*)node;
dcefreelist(target->segments);
} break;
case CES_FCN: {
DCEfcn* target = (DCEfcn*)node;
dcefreelist(target->args);
nullfree(target->name);
} break;
case CES_CONST: {
DCEconstant* target = (DCEconstant*)node;
if(target->discrim == CES_STR)
nullfree(target->text);
} break;
case CES_VALUE: {
DCEvalue* target = (DCEvalue*)node;
switch(target->discrim) {
case CES_CONST: dcefree((DCEnode*)target->constant); break;
case CES_VAR: dcefree((DCEnode*)target->var); break;
case CES_FCN: dcefree((DCEnode*)target->fcn); break;
default: assert(0);
}
} break;
case CES_PROJECT: {
DCEprojection* target = (DCEprojection*)node;
switch (target->discrim) {
case CES_VAR: dcefree((DCEnode*)target->var); break;
case CES_FCN: dcefree((DCEnode*)target->fcn); break;
default: assert(0);
}
} break;
case CES_SELECT: {
DCEselection* target = (DCEselection*)node;
dcefreelist(target->rhs);
dcefree((DCEnode*)target->lhs);
} break;
case CES_CONSTRAINT: {
DCEconstraint* target = (DCEconstraint*)node;
dcefreelist(target->projections);
dcefreelist(target->selections);
} break;
case CES_SEGMENT: {
DCEsegment* target = (DCEsegment*)node;
target->rank = 0;
nullfree(target->name);
} break;
case CES_SLICE: {
} break;
default:
assert(0);
}
/* final action */
free(node);
}
NCerror
fixprojections(NClist* list)
{
int i,j,k;
NCerror ncstat = NC_NOERR;
NClist* tmp = nclistnew(); /* misc. uses */
#ifdef DEBUG
fprintf(stderr,"fixprojection: list = %s\n",dumpprojections(list));
#endif
if(nclistlength(list) == 0) goto done;
/* Step 1: remove duplicates and complain about slice mismatches */
for(i=0;i<nclistlength(list);i++) {
DCEprojection* p1 = (DCEprojection*)nclistget(list,i);
if(p1 == NULL) continue;
if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */
for(j=i;j<nclistlength(list);j++) {
DCEprojection* p2 = (DCEprojection*)nclistget(list,j);
if(p2 == NULL) continue;
if(p1 == p2) continue;
if(p2->discrim != CES_VAR) continue;
if(p1->var->annotation != p2->var->annotation) continue;
/* check for slice mismatches */
if(!slicematch(p1->var->segments,p2->var->segments)) {
/* complain */
nclog(NCLOGWARN,"Malformed projection: same variable with different slicing");
}
/* remove p32 */
nclistset(list,j,(ncelem)NULL);
dcefree((DCEnode*)p2);
}
}
/* Step 2: remove containers when a field is also present */
for(i=0;i<nclistlength(list);i++) {
DCEprojection* p1 = (DCEprojection*)nclistget(list,i);
if(p1 == NULL) continue;
if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */
if(!iscontainer((CDFnode*)p1->var->annotation))
continue;
for(j=i;j<nclistlength(list);j++) {
DCEprojection* p2 = (DCEprojection*)nclistget(list,j);
if(p2 == NULL) continue;
if(p2->discrim != CES_VAR) continue;
nclistclear(tmp);
collectnodepath3((CDFnode*)p2->var->annotation,tmp,WITHDATASET);
for(k=0;k<nclistlength(tmp);k++) {
void* candidate = (void*)nclistget(tmp,k);
if(candidate == p1->var->annotation) {
nclistset(list,i,(ncelem)NULL);
dcefree((DCEnode*)p1);
goto next;
}
}
}
next: continue;
}
/* Step 3: expand all containers recursively down to the leaf nodes */
for(;;) {
nclistclear(tmp);
for(i=0;i<nclistlength(list);i++) {
DCEprojection* target = (DCEprojection*)nclistget(list,i);
CDFnode* leaf;
if(target == NULL) continue;
if(target->discrim != CES_VAR)
continue; /* dont try to unify functions */
leaf = (CDFnode*)target->var->annotation;
ASSERT(leaf != NULL);
if(iscontainer(leaf)) {/* capture container */
if(!nclistcontains(tmp,(ncelem)target))
nclistpush(tmp,(ncelem)target);
nclistset(list,i,(ncelem)NULL);
}
}
if(nclistlength(tmp) == 0) break; /*done*/
/* Now explode the containers */
for(i=0;i<nclistlength(tmp);i++) {
DCEprojection* container = (DCEprojection*)nclistget(tmp,i);
CDFnode* leaf = (CDFnode*)container->var->annotation;
for(j=0;i<nclistlength(leaf->subnodes);j++) {
CDFnode* field = (CDFnode*)nclistget(leaf->subnodes,j);
/* Convert field node to a proper constraint */
DCEprojection* proj = projectify(field,container);
nclistpush(list,(ncelem)proj);
}
/* reclaim the container */
dcefree((DCEnode*)container);
}
} /*for(;;)*/
/* remove all NULL elements */
for(i=nclistlength(list)-1;i>=0;i--) {
DCEprojection* target = (DCEprojection*)nclistget(list,i);
if(target == NULL)
nclistremove(list,i);
}
done:
#ifdef DEBUG
fprintf(stderr,"fixprojection: exploded = %s\n",dumpprojections(list));
#endif
nclistfree(tmp);
return ncstat;
}
NCerror
nc3d_getvarx(int ncid, int varid,
const size_t *startp,
const size_t *countp,
const ptrdiff_t* stridep,
void *data,
nc_type dsttype0)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
int i;
NC* drno;
NC* substrate;
NCDAPCOMMON* dapcomm;
CDFnode* cdfvar = NULL; /* cdf node mapping to var*/
NClist* varnodes;
nc_type dsttype;
Getvara* varainfo = NULL;
CDFnode* xtarget = NULL; /* target in DATADDS */
CDFnode* target = NULL; /* target in constrained DDS */
DCEprojection* varaprojection = NULL;
NCcachenode* cachenode = NULL;
size_t localcount[NC_MAX_VAR_DIMS];
NClist* ncdimsall;
size_t ncrank;
NClist* vars = NULL;
DCEconstraint* fetchconstraint = NULL;
DCEprojection* fetchprojection = NULL;
DCEprojection* walkprojection = NULL;
int state;
#define FETCHWHOLE 1 /* fetch whole data set */
#define FETCHVAR 2 /* fetch whole variable */
#define FETCHPART 4 /* fetch constrained variable */
#define CACHED 8 /* whole variable is already in the cache */
ncstat = NC_check_id(ncid, (NC**)&drno);
if(ncstat != NC_NOERR) goto fail;
dapcomm = (NCDAPCOMMON*)drno->dispatchdata;
ncstat = NC_check_id(drno->substrate, (NC**)&substrate);
if(ncstat != NC_NOERR) goto fail;
/* Locate var node via varid */
varnodes = dapcomm->cdf.ddsroot->tree->varnodes;
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(varnodes,i);
if(node->array.basevar == NULL
&& node->nctype == NC_Atomic
&& node->ncid == varid) {
cdfvar = node;
break;
}
}
ASSERT((cdfvar != NULL));
/* If the variable is prefetchable, then now
is the time to do a lazy prefetch */
if(FLAGSET(dapcomm->controls,NCF_PREFETCH)
&& !FLAGSET(dapcomm->controls,NCF_PREFETCH_EAGER)) {
if(dapcomm->cdf.cache != NULL && dapcomm->cdf.cache->prefetch == NULL) {
ncstat = prefetchdata3(dapcomm);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
}
}
/* Get the dimension info */
ncdimsall = cdfvar->array.dimsetall;
ncrank = nclistlength(ncdimsall);
#ifdef DEBUG
{
int i;
fprintf(stderr,"getvarx: %s",cdfvar->ncfullname);
for(i=0;i<ncrank;i++)
fprintf(stderr,"(%ld:%ld:%ld)",
(long)startp[i],
(long)countp[i],
(long)stridep[i]
);
fprintf(stderr,"\n");
}
#endif
/* Fill in missing arguments */
if(startp == NULL)
startp = nc_sizevector0;
if(countp == NULL) {
/* Accumulate the dimension sizes */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdimsall,i);
localcount[i] = dim->dim.declsize;
}
countp = localcount;
}
if(stridep == NULL)
stridep = nc_ptrdiffvector1;
/* Validate the dimension sizes */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdimsall,i);
if(startp[i] > dim->dim.declsize
|| startp[i]+countp[i] > dim->dim.declsize) {
ncstat = NC_EINVALCOORDS;
goto fail;
}
}
#ifdef DEBUG
{
NClist* dims = cdfvar->array.dimsetall;
fprintf(stderr,"getvarx: %s",cdfvar->ncfullname);
if(nclistlength(dims) > 0) {int i;
for(i=0;i<nclistlength(dims);i++)
fprintf(stderr,"(%lu:%lu:%lu)",(unsigned long)startp[i],(unsigned long)countp[i],(unsigned long)stridep[i]);
fprintf(stderr," -> ");
for(i=0;i<nclistlength(dims);i++)
if(stridep[i]==1)
fprintf(stderr,"[%lu:%lu]",(unsigned long)startp[i],(unsigned long)((startp[i]+countp[i])-1));
else {
unsigned long iend = (stridep[i] * countp[i]);
iend = (iend + startp[i]);
iend = (iend - 1);
fprintf(stderr,"[%lu:%lu:%lu]",
(unsigned long)startp[i],(unsigned long)stridep[i],iend);
}
}
fprintf(stderr,"\n");
}
#endif
dsttype = (dsttype0);
/* Default to using the inquiry type for this var*/
if(dsttype == NC_NAT) dsttype = cdfvar->externaltype;
/* Validate any implied type conversion*/
if(cdfvar->etype != dsttype && dsttype == NC_CHAR) {
/* The only disallowed conversion is to/from char and non-byte
numeric types*/
switch (cdfvar->etype) {
case NC_STRING: case NC_URL:
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
break;
default:
return THROW(NC_ECHAR);
}
}
ncstat = makegetvar34(dapcomm,cdfvar,data,dsttype,&varainfo);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Compile the start/stop/stride info into a projection */
ncstat = buildvaraprojection3(varainfo->target,
startp,countp,stridep,
&varaprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
fetchprojection = NULL;
walkprojection = NULL;
/* Create walkprojection as the merge of the url projections
and the vara projection; may change in FETCHPART case below*/
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&walkprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
#ifdef DEBUG
fprintf(stderr,"getvarx: walkprojection: |%s|\n",dumpprojection(walkprojection));
#endif
/* define the var list of interest */
vars = nclistnew();
nclistpush(vars,(void*)varainfo->target);
state = 0;
if(iscached(dapcomm,cdfvar,&cachenode)) { /* ignores non-whole variable cache entries */
state = CACHED;
ASSERT((cachenode != NULL));
#ifdef DEBUG
fprintf(stderr,"var is in cache\n");
#endif
/* If it is cached, then it is a whole variable but may still
need to apply constraints during the walk */
ASSERT(cachenode->wholevariable); /* by construction */
} else if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
state = FETCHWHOLE;
} else {/* load using constraints */
if(FLAGSET(dapcomm->controls,NCF_WHOLEVAR))
state = FETCHVAR;
else
state = FETCHPART;
}
ASSERT(state != 0);
switch (state) {
case FETCHWHOLE: {
/* buildcachenode3 will create a new cachenode and
will also fetch the whole corresponding datadds.
*/
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* Use no projections or selections */
fetchconstraint->projections = nclistnew();
fetchconstraint->selections = nclistnew();
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHWHOLE: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
case CACHED: {
} break;
case FETCHVAR: { /* Fetch a complete single variable */
/* Create fetch projection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
/* elide any sequence and string dimensions (dap servers do not allow such). */
ncstat = removepseudodims(fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Convert to a whole variable projection */
dcemakewholeprojection(fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHVAR: fetchprojection: |%s|\n",dumpprojection(fetchprojection));
#endif
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* merged constraint just uses the url constraint selection */
fetchconstraint->selections = dceclonelist(dapcomm->oc.dapconstraint->selections);
/* and the created fetch projection */
fetchconstraint->projections = nclistnew();
nclistpush(fetchconstraint->projections,(void*)fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHVAR: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
/* buildcachenode3 will create a new cachenode and
will also fetch the corresponding datadds.
*/
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
case FETCHPART: {
/* Create fetch projection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
/* elide any sequence and string dimensions (dap servers do not allow such). */
ncstat = removepseudodims(fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Shift the varaprojection for simple walk */
dcefree((DCEnode*)walkprojection) ; /* reclaim any existing walkprojection */
walkprojection = (DCEprojection*)dceclone((DCEnode*)varaprojection);
dapshiftprojection(walkprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHPART: fetchprojection: |%s|\n",dumpprojection(fetchprojection));
#endif
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* merged constraint just uses the url constraint selection */
fetchconstraint->selections = dceclonelist(dapcomm->oc.dapconstraint->selections);
/* and the created fetch projection */
fetchconstraint->projections = nclistnew();
nclistpush(fetchconstraint->projections,(void*)fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHPART: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
/* buildcachenode3 will create a new cachenode and
will also fetch the corresponding datadds.
*/
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
default: PANIC1("unknown fetch state: %d\n",state);
}
ASSERT(cachenode != NULL);
#ifdef DEBUG
fprintf(stderr,"cache.datadds=%s\n",dumptree(cachenode->datadds));
#endif
/* attach DATADDS to (constrained) DDS */
unattach34(dapcomm->cdf.ddsroot);
ncstat = attachsubset34(cachenode->datadds,dapcomm->cdf.ddsroot);
if(ncstat) goto fail;
/* Fix up varainfo to use the cache */
varainfo->cache = cachenode;
cachenode = NULL;
varainfo->varaprojection = walkprojection;
walkprojection = NULL;
/* Get the var correlate from the datadds */
target = varainfo->target;
xtarget = target->attachment;
if(xtarget == NULL)
{THROWCHK(ncstat=NC_ENODATA); goto fail;}
/* Switch to datadds tree space*/
varainfo->target = xtarget;
save = (DCEnode*)varaprojection;
ncstat = moveto(dapcomm,varainfo,varainfo->cache->datadds,data);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
nclistfree(vars);
dcefree((DCEnode*)varaprojection);
dcefree((DCEnode*)fetchconstraint);
freegetvara(varainfo);
fail:
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
/* Given the arguments to vara
construct a corresponding projection
with any pseudo dimensions removed
*/
NCerror
buildvaraprojection(CDFnode* var,
const size_t* startp, const size_t* countp, const ptrdiff_t* stridep,
DCEprojection** projectionp)
{
int i,j;
NCerror ncstat = NC_NOERR;
DCEprojection* projection = NULL;
NClist* path = nclistnew();
NClist* segments = NULL;
int dimindex;
/* Build a skeleton projection that has 1 segment for
every cdfnode from root to the variable of interest.
Each segment has the slices from its corresponding node
in the path, including pseudo-dims
*/
ncstat = dapvar2projection(var,&projection);
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection: skeleton: %s\n",dumpprojection(projection));
#endif
/* Now, modify the projection to reflect the corresponding
start/count/stride from the nc_get_vara arguments.
*/
segments = projection->var->segments;
dimindex = 0;
for(i=0;i<nclistlength(segments);i++) {
DCEsegment* segment = (DCEsegment*)nclistget(segments,i);
for(j=0;j<segment->rank;j++) {
size_t count = 0;
DCEslice* slice = &segment->slices[j];
/* make each slice represent the corresponding
start/count/stride */
slice->first = startp[dimindex+j];
slice->stride = stridep[dimindex+j];
count = countp[dimindex+j];
slice->count = count;
slice->length = count * slice->stride;
slice->last = (slice->first + slice->length) - 1;
if(slice->last >= slice->declsize) {
slice->last = slice->declsize - 1;
/* reverse compute the new length */
slice->length = (slice->last - slice->first) + 1;
}
}
dimindex += segment->rank;
}
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection.final: %s\n",dumpprojection(projection));
#endif
#ifdef DEBUG
fprintf(stderr,"buildvaraprojection3: final: projection=%s\n",
dumpprojection(projection));
#endif
if(projectionp) *projectionp = projection;
nclistfree(path);
if(ncstat) dcefree((DCEnode*)projection);
return ncstat;
}
/* Compute the set of prefetched data.
Notes:
1. All prefetches are whole variable fetches.
2. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i;
NCFLAGS flags;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else {
/* pull in those variables previously marked as prefetchable */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
/* Most of the important testing was already done */
if(!var->basenode->prefetchable)
continue;
/* Do not attempt to prefetch any variables in the
nc_open url's projection list
*/
if(nclistcontains(nccomm->cdf.projectedvars,(void*)var))
continue;
/* Should be prefetchable */
nclistpush(vars,(void*)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
The exception is if we are prefetching everything.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(void*)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
flags = NCF_PREFETCH;
if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL;
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
NCerror
buildcachenode34(NCDAPCOMMON* nccomm,
DCEconstraint* constraint,
NClist* varlist,
NCcachenode** cachep,
NCFLAGS flags)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
OClink conn = nccomm->oc.conn;
OCddsnode ocroot = NULL;
CDFnode* dxdroot = NULL;
NCcachenode* cachenode = NULL;
char* ce = NULL;
int isprefetch = 0;
if((flags & NCF_PREFETCH) != 0)
isprefetch = 1;
if((flags & NCF_PREFETCH_ALL) == 0)
ce = buildconstraintstring3(constraint);
ocstat = dap_fetch(nccomm,conn,ce,OCDATADDS,&ocroot);
nullfree(ce);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
ncstat = buildcdftree34(nccomm,ocroot,OCDATA,&dxdroot);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* re-struct*/
if(!FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
ncstat = restruct3(dxdroot,nccomm->cdf.ddsroot,constraint->projections);
if(ncstat) {THROWCHK(ncstat); goto done;}
}
/* create the cache node */
cachenode = createnccachenode();
cachenode->isprefetch = isprefetch;
cachenode->vars = nclistclone(varlist);
cachenode->datadds = dxdroot;
/* Give the constraint over to the cachenode */
cachenode->constraint = constraint;
constraint = NULL;
cachenode->wholevariable = iscacheableconstraint(cachenode->constraint);
/* save the root content*/
cachenode->ocroot = ocroot;
ocstat = oc_data_getroot(conn,ocroot,&cachenode->content);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
/* capture the packet size */
ocstat = oc_raw_xdrsize(conn,ocroot,&cachenode->xdrsize);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
#ifdef DEBUG
fprintf(stderr,"buildcachenode: new cache node: %s\n",
dumpcachenode(cachenode));
#endif
/* Insert into the cache. If not caching, then
remove any previous cache node
*/
if(!isprefetch) {
NCcache* cache = nccomm->cdf.cache;
if(cache->nodes == NULL) cache->nodes = nclistnew();
/* remove cache nodes to get below the max cache size */
while(cache->cachesize + cachenode->xdrsize > cache->cachelimit
&& nclistlength(cache->nodes) > 0) {
NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0);
#ifdef DEBUG
fprintf(stderr,"buildcachenode: purge cache node: %s\n",
dumpcachenode(cachenode));
#endif
cache->cachesize -= node->xdrsize;
freenccachenode(nccomm,node);
}
/* Remove cache nodes to get below the max cache count */
/* If not caching, then cachecount should be 0 */
while(nclistlength(cache->nodes) > cache->cachecount) {
NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0);
#ifdef DEBUG
fprintf(stderr,"buildcachenode: count purge cache node: %s\n",
dumpcachenode(node));
#endif
cache->cachesize -= node->xdrsize;
freenccachenode(nccomm,node);
}
nclistpush(nccomm->cdf.cache->nodes,(void*)cachenode);
cache->cachesize += cachenode->xdrsize;
}
#ifdef DEBUG
fprintf(stderr,"buildcachenode: %s\n",dumpcachenode(cachenode));
#endif
done:
if(constraint != NULL) dcefree((DCEnode*)constraint);
if(cachep) *cachep = cachenode;
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
if(ncstat) {
freecdfroot34(dxdroot);
freenccachenode(nccomm,cachenode);
}
return THROW(ncstat);
}
/* Compute the set of prefetched data.
Notes:
1. Even if caching is off, we will
still prefetch the small variables.
2. All prefetches are whole variable fetches.
3. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
int isnc4 = FLAGSET(nccomm->controls,NCF_NC4);
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
if(!isnc4) {
/* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
}
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimset0);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j);
nelems *= dim->dim.declsize;
}
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems);
}
if(nelems <= nccomm->cdf.smallsizelimit) {
nclistpush(vars,(ncelem)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(ncelem)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
NCerror
buildvaraprojection3(Getvara* getvar,
const size_t* startp, const size_t* countp, const ptrdiff_t* stridep,
DCEprojection** projectionp)
{
int i,j;
int ncrank;
NCerror ncstat = NC_NOERR;
CDFnode* var = getvar->target;
NClist* vardims = var->array.dimensions;
DCEprojection* projection = NULL;
NClist* path = nclistnew();
NClist* segments = NULL;
int dimindex;
NClist* dimset = NULL;
/* Collect the nodes needed to construct the projection segment */
collectnodepath3(var,path,!WITHDATASET);
dimset = var->array.dimensions;
ncrank = nclistlength(dimset);
segments = nclistnew();
nclistsetalloc(segments,nclistlength(path));
for(i=0;i<nclistlength(path);i++) {
DCEsegment* segment = (DCEsegment*)dcecreate(CES_SEGMENT);
CDFnode* n = (CDFnode*)nclistget(path,i);
segment->cdfnode = n;
ASSERT((segment->cdfnode != NULL));
segment->name = nulldup(n->name);
segment->slicesdefined = 0; /* temporary */
segment->slicesdeclized = 0; /* temporary */
nclistpush(segments,(ncelem)segment);
}
projection = (DCEprojection*)dcecreate(CES_PROJECT);
projection->discrim = CES_VAR;
projection->var = (DCEvar*)dcecreate(CES_VAR);
projection->var->cdfleaf = var;
projection->var->segments = segments;
/* We need to assign slices to each segment */
dimindex = 0; /* point to next subset of slices */
for(i=0;i<nclistlength(segments);i++) {
DCEsegment* segment = (DCEsegment*)nclistget(segments,i);
int localrank = nclistlength(segment->cdfnode->array.dimensions0);
if(segment->cdfnode->array.stringdim != NULL) localrank++;
if(segment->cdfnode->array.seqdim != NULL) localrank++;
segment->rank = localrank;
for(j=0;j<localrank;j++) {
DCEslice* slice = &segment->slices[j];
CDFnode* dim = (CDFnode*)nclistget(vardims,dimindex+j);
slice->first = startp[dimindex+j];
slice->stride = stridep[dimindex+j];
slice->count = countp[dimindex+j];
slice->length = slice->count * slice->stride;
if(slice->length > dim->dim.declsize)
slice->length = dim->dim.declsize;
slice->stop = (slice->first + slice->length);
if(slice->stop > dim->dim.declsize)
slice->stop = dim->dim.declsize;
slice->declsize = dim->dim.declsize;
}
segment->slicesdefined = 1;
segment->slicesdeclized = 1;
dimindex += localrank;
ASSERT((dimindex <= ncrank));
}
ASSERT((dimindex == ncrank));
if(projectionp) *projectionp = projection;
nclistfree(path);
if(ncstat) dcefree((DCEnode*)projection);
return ncstat;
}
void
restrictprojection34(NClist* varlist, NClist* projections)
{
int i,j,len;
#ifdef DEBUG
fprintf(stderr,"restriction.before=|%s|\n",
dumpprojections(projections));
#endif
if(nclistlength(varlist) == 0) goto done; /* nothing to add or remove */
/* If the projection list is empty, then add
a projection for every variable in varlist
*/
if(nclistlength(projections) == 0) {
NClist* path = nclistnew();
NClist* nodeset = NULL;
/* Attempt to unify the vars into larger units
(like a complete grid) */
nodeset = unifyprojectionnodes3(varlist);
for(i=0;i<nclistlength(nodeset);i++) {
CDFnode* var = (CDFnode*)nclistget(nodeset,i);
#ifdef DEBUG
fprintf(stderr,"restriction.candidate=|%s|\n",var->ncfullname);
#endif
DCEprojection* newp = (DCEprojection*)dcecreate(CES_PROJECT);
newp->discrim = CES_VAR;
newp->var = (DCEvar*)dcecreate(CES_VAR);
newp->var->cdfleaf = var;
nclistclear(path);
collectnodepath3(var,path,!WITHDATASET);
newp->var->segments = nclistnew();
for(j=0;j<nclistlength(path);j++) {
CDFnode* node = (CDFnode*)nclistget(path,j);
DCEsegment* newseg = (DCEsegment*)dcecreate(CES_SEGMENT);
newseg->name = nulldup(node->name);
makewholesegment3(newseg,node);/*treat as simple projections*/
newseg->cdfnode = node;
nclistpush(newp->var->segments,(ncelem)newseg);
}
nclistpush(projections,(ncelem)newp);
}
nclistfree(path);
nclistfree(nodeset);
} else {
/* Otherwise, walk all the projections and see if they
intersect any of the variables. If not,
then remove from the projection list.
*/
len = nclistlength(projections);
for(i=len-1;i>=0;i--) {/* Walk backward to facilitate removal*/
int intersect = 0;
DCEprojection* proj = (DCEprojection*)nclistget(projections,i);
if(proj->discrim != CES_VAR) continue;
for(j=0;j<nclistlength(varlist);j++) {
CDFnode* var = (CDFnode*)nclistget(varlist,j);
/* Note that intersection could go either way */
if(treecontains3(var,proj->var->cdfleaf)
|| treecontains3(proj->var->cdfleaf,var)) {intersect = 1; break;}
}
if(!intersect) {
/* suppress this projection */
DCEprojection* p = (DCEprojection*)nclistremove(projections,i);
dcefree((DCEnode*)p);
}
}
/* Now looks for containment between projections and only keep
the more restrictive. Is this algorithm stable against reordering?.
*/
for(;;) {
int removed = 0;
for(i=0;i<nclistlength(projections);i++) {
DCEprojection* pi = (DCEprojection*)nclistget(projections,i);
if(pi->discrim != CES_VAR) continue;
for(j=0;j<i;j++) {
DCEprojection* pj = (DCEprojection*)nclistget(projections,j);
if(pj->discrim != CES_VAR) continue;
if(treecontains3(pi->var->cdfleaf,pj->var->cdfleaf)) {
DCEprojection* p = (DCEprojection*)nclistremove(projections,j);
dcefree((DCEnode*)p);
removed = 1;
break;
} else if(treecontains3(pj->var->cdfleaf,pi->var->cdfleaf)) {
DCEprojection* p = (DCEprojection*)nclistremove(projections,i);
dcefree((DCEnode*)p);
removed = 1;
break;
}
}
}
if(!removed) break;
}
}
done:
#ifdef DEBUG
fprintf(stderr,"restriction.after=|%s|\n",
dumpprojections(projections));
#endif
return;
}
