/* Compute the set of variables referenced in the projections
of the input constraint.
*/
NCerror
computeprojectedvars(NCDAPCOMMON* dapcomm, DCEconstraint* constraint)
{
NCerror ncstat = NC_NOERR;
NClist* vars = NULL;
int i;
vars = nclistnew();
if(dapcomm->cdf.projectedvars != NULL)
nclistfree(dapcomm->cdf.projectedvars);
dapcomm->cdf.projectedvars = vars;
if(constraint == NULL || constraint->projections == NULL)
goto done;
for(i=0;i<nclistlength(constraint->projections);i++) {
CDFnode* node;
DCEprojection* proj = (DCEprojection*)nclistget(constraint->projections,i);
if(proj->discrim == CES_FCN) continue; /* ignore these */
node = (CDFnode*)proj->var->annotation;
if(!nclistcontains(vars,(void*)node)) {
nclistpush(vars,(void*)node);
}
}
done:
return ncstat;
}
static void
ceallnodesr(DCEnode* node, NClist* allnodes, CEsort which)
{
int i;
if(node == NULL) return;
if(nclistcontains(allnodes,(ncelem)node)) return;
if(which == CES_NIL || node->sort == which)
nclistpush(allnodes,(ncelem)node);
switch(node->sort) {
case CES_FCN: {
DCEfcn* fcn = (DCEfcn*)node;
for(i=0;i<nclistlength(fcn->args);i++) {
ceallnodesr((DCEnode*)nclistget(fcn->args,i),allnodes,which);
}
} break;
case CES_VAR: {
DCEvar* var = (DCEvar*)node;
for(i=0;i<nclistlength(var->segments);i++) {
ceallnodesr((DCEnode*)nclistget(var->segments,i),allnodes,which);
}
} break;
case CES_VALUE: {
DCEvalue* value = (DCEvalue*)node;
if(value->discrim == CES_VAR)
ceallnodesr((DCEnode*)value->var,allnodes,which);
else if(value->discrim == CES_FCN)
ceallnodesr((DCEnode*)value->fcn,allnodes,which);
else
ceallnodesr((DCEnode*)value->constant,allnodes,which);
} break;
case CES_SELECT: {
DCEselection* selection = (DCEselection*)node;
ceallnodesr((DCEnode*)selection->lhs,allnodes,which);
for(i=0;i<nclistlength(selection->rhs);i++)
ceallnodesr((DCEnode*)nclistget(selection->rhs,i),allnodes,which);
} break;
case CES_PROJECT: {
DCEprojection* projection = (DCEprojection*)node;
if(projection->discrim == CES_VAR)
ceallnodesr((DCEnode*)projection->var,allnodes,which);
else
ceallnodesr((DCEnode*)projection->fcn,allnodes,which);
} break;
case CES_CONSTRAINT: {
DCEconstraint* constraint = (DCEconstraint*)node;
for(i=0;i<nclistlength(constraint->projections);i++)
ceallnodesr((DCEnode*)nclistget(constraint->projections,i),allnodes,which);
for(i=0;i<nclistlength(constraint->selections);i++)
ceallnodesr((DCEnode*)nclistget(constraint->selections,i),allnodes,which);
} break;
/* All others have no subnodes */
default:
break;
}
}
static NCerror
sequencecheckr(CDFnode* node, NClist* vars, CDFnode* topseq)
{
unsigned int i;
NCerror err = NC_NOERR;
int ok = 0;
if(topseq == NULL && nclistlength(node->array.dimset0) > 0) {
err = NC_EINVAL; /* This container has dimensions, so no sequence within it
can be usable */
} else if(node->nctype == NC_Sequence) {
/* Recursively walk the path for each subnode of this sequence node
looking for a path without any sequence */
for(i=0; i<nclistlength(node->subnodes); i++) {
CDFnode* sub = (CDFnode*)nclistget(node->subnodes,i);
err = sequencecheckr(sub,vars,node);
if(err == NC_NOERR) ok = 1; /* there is at least 1 usable var below */
}
if(topseq == NULL && ok == 1) {
/* this sequence is usable because it has scalar container
(by construction) and has a path to a leaf without an intermediate
sequence. */
err = NC_NOERR;
node->usesequence = 1;
} else {
/* this sequence is unusable because it has no path
to a leaf without an intermediate sequence. */
node->usesequence = 0;
err = NC_EINVAL;
}
} else if(nclistcontains(vars,(void*)node)) {
/* If we reach a leaf, then topseq is usable, so save it */
node->array.sequence = topseq;
} else { /* Some kind of non-sequence container node with no dimensions */
/* recursively compute usability */
for(i=0; i<nclistlength(node->subnodes); i++) {
CDFnode* sub = (CDFnode*)nclistget(node->subnodes,i);
err = sequencecheckr(sub,vars,topseq);
if(err == NC_NOERR) ok = 1;
}
err = (ok?NC_NOERR:NC_EINVAL);
}
return err;
}
NCerror
fixprojections(NClist* list)
{
int i,j,k;
NCerror ncstat = NC_NOERR;
NClist* tmp = nclistnew(); /* misc. uses */
#ifdef DEBUG
fprintf(stderr,"fixprojection: list = %s\n",dumpprojections(list));
#endif
if(nclistlength(list) == 0) goto done;
/* Step 1: remove duplicates and complain about slice mismatches */
for(i=0;i<nclistlength(list);i++) {
DCEprojection* p1 = (DCEprojection*)nclistget(list,i);
if(p1 == NULL) continue;
if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */
for(j=i;j<nclistlength(list);j++) {
DCEprojection* p2 = (DCEprojection*)nclistget(list,j);
if(p2 == NULL) continue;
if(p1 == p2) continue;
if(p2->discrim != CES_VAR) continue;
if(p1->var->annotation != p2->var->annotation) continue;
/* check for slice mismatches */
if(!slicematch(p1->var->segments,p2->var->segments)) {
/* complain */
nclog(NCLOGWARN,"Malformed projection: same variable with different slicing");
}
/* remove p32 */
nclistset(list,j,(ncelem)NULL);
dcefree((DCEnode*)p2);
}
}
/* Step 2: remove containers when a field is also present */
for(i=0;i<nclistlength(list);i++) {
DCEprojection* p1 = (DCEprojection*)nclistget(list,i);
if(p1 == NULL) continue;
if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */
if(!iscontainer((CDFnode*)p1->var->annotation))
continue;
for(j=i;j<nclistlength(list);j++) {
DCEprojection* p2 = (DCEprojection*)nclistget(list,j);
if(p2 == NULL) continue;
if(p2->discrim != CES_VAR) continue;
nclistclear(tmp);
collectnodepath3((CDFnode*)p2->var->annotation,tmp,WITHDATASET);
for(k=0;k<nclistlength(tmp);k++) {
void* candidate = (void*)nclistget(tmp,k);
if(candidate == p1->var->annotation) {
nclistset(list,i,(ncelem)NULL);
dcefree((DCEnode*)p1);
goto next;
}
}
}
next: continue;
}
/* Step 3: expand all containers recursively down to the leaf nodes */
for(;;) {
nclistclear(tmp);
for(i=0;i<nclistlength(list);i++) {
DCEprojection* target = (DCEprojection*)nclistget(list,i);
CDFnode* leaf;
if(target == NULL) continue;
if(target->discrim != CES_VAR)
continue; /* dont try to unify functions */
leaf = (CDFnode*)target->var->annotation;
ASSERT(leaf != NULL);
if(iscontainer(leaf)) {/* capture container */
if(!nclistcontains(tmp,(ncelem)target))
nclistpush(tmp,(ncelem)target);
nclistset(list,i,(ncelem)NULL);
}
}
if(nclistlength(tmp) == 0) break; /*done*/
/* Now explode the containers */
for(i=0;i<nclistlength(tmp);i++) {
DCEprojection* container = (DCEprojection*)nclistget(tmp,i);
CDFnode* leaf = (CDFnode*)container->var->annotation;
for(j=0;i<nclistlength(leaf->subnodes);j++) {
CDFnode* field = (CDFnode*)nclistget(leaf->subnodes,j);
/* Convert field node to a proper constraint */
DCEprojection* proj = projectify(field,container);
nclistpush(list,(ncelem)proj);
}
/* reclaim the container */
dcefree((DCEnode*)container);
}
} /*for(;;)*/
/* remove all NULL elements */
for(i=nclistlength(list)-1;i>=0;i--) {
DCEprojection* target = (DCEprojection*)nclistget(list,i);
if(target == NULL)
nclistremove(list,i);
}
done:
#ifdef DEBUG
fprintf(stderr,"fixprojection: exploded = %s\n",dumpprojections(list));
#endif
nclistfree(tmp);
return ncstat;
}
/* Compute the set of prefetched data.
Notes:
1. All prefetches are whole variable fetches.
2. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i;
NCFLAGS flags;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else {
/* pull in those variables previously marked as prefetchable */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
/* Most of the important testing was already done */
if(!var->basenode->prefetchable)
continue;
/* Do not attempt to prefetch any variables in the
nc_open url's projection list
*/
if(nclistcontains(nccomm->cdf.projectedvars,(void*)var))
continue;
/* Should be prefetchable */
nclistpush(vars,(void*)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
The exception is if we are prefetching everything.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(void*)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
flags = NCF_PREFETCH;
if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL;
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
/* See if we can unify sets of nodes to be projected
into larger units.
*/
static NClist*
unifyprojectionnodes3(NClist* varlist)
{
int i;
NClist* nodeset = nclistnew();
NClist* containerset = nclistnew();
NClist* containernodes = nclistnew();
nclistsetalloc(nodeset,nclistlength(varlist));
nclistsetalloc(containerset,nclistlength(varlist));
/* Duplicate the varlist so we can modify it;
simultaneously collect unique container set.
*/
for(i=0;i<nclistlength(varlist);i++) {
CDFnode* var = (CDFnode*)nclistget(varlist,i);
CDFnode* container = var->container;
nclistpush(nodeset,(ncelem)var);
switch (container->nctype) {
case NC_Sequence: case NC_Structure: case NC_Grid: case NC_Dataset:
/* add (uniquely) to container set */
if(!nclistcontains(containerset,(ncelem)container))
nclistpush(containerset,(ncelem)container);
break;
default: break;
}
}
/* Now, try to find containers whose subnodes are all in the
varlist; repeat until no more changes */
for(;;) {
int changed = 0;
for(i=0;i<nclistlength(containerset);i++) {
int j, allfound;
CDFnode* container = (CDFnode*)nclistget(containerset,i);
if(container == NULL) continue;
nclistclear(containernodes);
for(allfound=1,j=0;j<nclistlength(container->subnodes);j++) {
CDFnode* subnode = (CDFnode*)nclistget(container->subnodes,j);
if(!nclistcontains(varlist,(ncelem)subnode)) {allfound=0;break;}
nclistpush(containernodes,(ncelem)subnode);
}
if(allfound) {
nclistpush(nodeset,(ncelem)container);
nclistset(containerset,i,(ncelem)NULL); /* remove */
for(j=nclistlength(nodeset)-1;j>=0;j--) { /* walk backwards */
CDFnode* testnode = (CDFnode*)nclistget(nodeset,j);
if(nclistcontains(containernodes,(ncelem)testnode))
nclistremove(nodeset,j);/* remove */
}
changed = 1;
}
}
if(!changed) break; /* apparently we have reached a stable situation */
}
/* If there is only the dataset left as a projection, then remove it */
if(nclistlength(nodeset) == 1) {
CDFnode* thenode = (CDFnode*)nclistget(nodeset,0);
if(thenode->nctype == NC_Dataset) nclistclear(nodeset);
}
nclistfree(containerset);
nclistfree(containernodes);
return nodeset;
}