/*
Given two projection lists, merge
src into dst taking
overlapping projections into acct.
Assume that name qualification has occured.
Dst will be modified.
*/
NCerror
mergeprojections3(NClist* dst, NClist* src)
{
int i;
NClist* cat = nclistnew();
NCerror ncstat = NC_NOERR;
#ifdef DEBUG
fprintf(stderr,"mergeprojection: dst = %s\n",dumpprojections(dst));
fprintf(stderr,"mergeprojection: src = %s\n",dumpprojections(src));
#endif
ASSERT(dst != NULL);
/* get dst concat clone(src) */
nclistsetalloc(cat,nclistlength(dst)+nclistlength(src));
for(i=0;i<nclistlength(dst);i++) {
DCEprojection* p = (DCEprojection*)nclistget(dst,i);
nclistpush(cat,(ncelem)p);
}
if(src != NULL) for(i=0;i<nclistlength(src);i++) {
DCEprojection* p = (DCEprojection*)nclistget(src,i);
nclistpush(cat,(ncelem)dceclone((DCEnode*)p));
}
nclistclear(dst);
/* Repeatedly pull elements from the concat,
merge with all duplicates, and stick into
the dst
*/
while(nclistlength(cat) > 0) {
DCEprojection* target = (DCEprojection*)nclistremove(cat,0);
if(target == NULL) continue;
if(target->discrim != CES_VAR) continue;
for(i=0;i<nclistlength(cat);i++) {
DCEprojection* p2 = (DCEprojection*)nclistget(cat,i);
if(p2 == NULL) continue;
if(p2->discrim != CES_VAR) continue;
if(target->var->cdfleaf != p2->var->cdfleaf) continue;
/* This entry matches our current target; merge */
ncstat = mergeprojection31(target,p2);
/* null out this merged entry and release it */
nclistset(cat,i,(ncelem)NULL);
dcefree((DCEnode*)p2);
}
/* Capture the clone */
nclistpush(dst,(ncelem)target);
}
nclistfree(cat);
return ncstat;
}
NCerror
mapconstraints3(DCEconstraint* constraint,
CDFnode* root)
{
int i;
NCerror ncstat = NC_NOERR;
NClist* nodes = root->tree->nodes;
NClist* dceprojections;
NClist* dceselections;
dceprojections = constraint->projections;
dceselections = constraint->selections;
/* Convert the projection paths to leaves in the dds tree */
for(i=0;i<nclistlength(dceprojections);i++) {
DCEprojection* proj = (DCEprojection*)nclistget(dceprojections,i);
if(proj->discrim != CES_VAR) continue; // ignore functions
ncstat = matchpartialname3(nodes,proj->var->segments,
(CDFnode**)&proj->var->annotation);
if(ncstat) goto done;
}
#ifdef IGNORE
Only care about projections
/* Convert the selection paths to leaves in the dds tree */
for(i=0;i<nclistlength(dceselections);i++) {
DCEselection* sel = (DCEselection*)nclistget(dceselections,i);
if(sel->lhs->discrim != CES_VAR) continue;
ncstat = matchpartialname3(nodes,sel->lhs->var->segments,
(CDFnode**)&sel->lhs->var->annotation);
if(ncstat) goto done;
}
/* Convert the selection path values to leaves in the dds tree */
for(i=0;i<nclistlength(dceselections);i++) {
int j;
DCEselection* sel = (DCEselection*)nclistget(dceselections,i);
for(j=0;j<nclistlength(sel->rhs);j++) {
DCEvalue* value = (DCEvalue*)nclistget(sel->rhs,j);
if(value->discrim != CES_VAR) continue;
ncstat = matchpartialname3(nodes,value->var->segments,
(CDFnode**)&value->var->annotation);
if(ncstat) goto done;
}
}
#endif
#ifdef DEBUG
fprintf(stderr,"mapconstraint.projections: %s\n",
dumpprojections(dceprojections));
fprintf(stderr,"mapconstraint.selections: %s\n",
dumpselections(dceselections));
#endif
done:
return THROW(ncstat);
}
NCerror
mapconstraints3(NCDAPCOMMON* dapcomm)
{
int i;
NCerror ncstat = NC_NOERR;
CDFnode* root = dapcomm->cdf.ddsroot;
NClist* nodes = root->tree->nodes;
NClist* dceprojections;
NClist* dceselections;
dceprojections = dapcomm->oc.dapconstraint->projections;
dceselections = dapcomm->oc.dapconstraint->selections;
/* Convert the projection paths to leaves in the dds tree */
for(i=0;i<nclistlength(dceprojections);i++) {
DCEprojection* proj = (DCEprojection*)nclistget(dceprojections,i);
if(proj->discrim != CES_VAR) continue;
ncstat = matchpartialname3(nodes,proj->var->segments,
&proj->var->cdfleaf);
if(ncstat) goto done;
}
/* Convert the selection paths to leaves in the dds tree */
for(i=0;i<nclistlength(dceselections);i++) {
DCEselection* sel = (DCEselection*)nclistget(dceselections,i);
if(sel->lhs->discrim != CES_VAR) continue;
ncstat = matchpartialname3(nodes,sel->lhs->var->segments,&sel->lhs->var->cdfleaf);
if(ncstat) goto done;
}
/* Convert the selection path values to leaves in the dds tree */
for(i=0;i<nclistlength(dceselections);i++) {
int j;
DCEselection* sel = (DCEselection*)nclistget(dceselections,i);
for(j=0;j<nclistlength(sel->rhs);j++) {
DCEvalue* value = (DCEvalue*)nclistget(sel->rhs,j);
if(value->discrim != CES_VAR) continue;
ncstat = matchpartialname3(nodes,value->var->segments,&value->var->cdfnode);
if(ncstat) goto done;
}
}
/* Fill in segment information */
ncstat = qualifyconstraints3(dapcomm->oc.dapconstraint);
if(ncstat != NC_NOERR) goto done;
#ifdef DEBUG
fprintf(stderr,"mapconstraint.projections: %s\n",
dumpprojections(dceprojections));
fprintf(stderr,"mapconstraint.selections: %s\n",
dumpselections(dceselections));
#endif
done:
return THROW(ncstat);
}
/*
Given a set of projections and a projection
representing a variable (from, say vara or prefetch)
construct a single projection for fetching that variable
with the proper constraints.
*/
int
daprestrictprojection(NClist* projections, DCEprojection* var, DCEprojection** resultp)
{
int ncstat = NC_NOERR;
int i;
DCEprojection* result = NULL;
#ifdef DEBUG1
fprintf(stderr,"restrictprojection.before: constraints=|%s| vara=|%s|\n",
dumpprojections(projections),
dumpprojection(var));
#endif
ASSERT(var != NULL);
/* the projection list will contain at most 1 match for the var by construction */
for(result=null,i=0;i<nclistlength(projections);i++) {
DCEprojection* p1 = (DCEprojection*)nclistget(projections,i);
if(p1 == NULL || p1->discrim != CES_VAR) continue;
if(p1->var->annotation == var->var->annotation) {
result = p1;
break;
}
}
if(result == NULL) {
result = (DCEprojection*)dceclone((DCEnode*)var); /* use only the var projection */
goto done;
}
result = (DCEprojection*)dceclone((DCEnode*)result); /* so we can modify */
#ifdef DEBUG1
fprintf(stderr,"restrictprojection.choice: base=|%s| add=|%s|\n",
dumpprojection(result),dumpprojection(var));
#endif
/* We need to merge the projection from the projection list
with the var projection
*/
ncstat = dcemergeprojections(result,var); /* result will be modified */
done:
if(resultp) *resultp = result;
#ifdef DEBUG
fprintf(stderr,"restrictprojection.after=|%s|\n",
dumpprojection(result));
#endif
return ncstat;
}
char*
dumpconstraint(NCconstraint* con)
{
NCbytes* buf = ncbytesnew();
char* result = NULL;
if(nclistlength(con->projections)==0 && nclistlength(con->selections)==0)
goto done;
if(nclistlength(con->projections) > 0) {
char* pstring = dumpprojections(con->projections);
ncbytescat(buf,pstring);
efree(pstring);
}
if(nclistlength(con->selections) > 0) {
char* sstring = dumpselections(con->selections);
ncbytescat(buf,sstring);
efree(sstring);
}
done:
result = ncbytesdup(buf);
ncbytesfree(buf);
return result;
}
NCerror
fixprojections(NClist* list)
{
int i,j,k;
NCerror ncstat = NC_NOERR;
NClist* tmp = nclistnew(); /* misc. uses */
#ifdef DEBUG
fprintf(stderr,"fixprojection: list = %s\n",dumpprojections(list));
#endif
if(nclistlength(list) == 0) goto done;
/* Step 1: remove duplicates and complain about slice mismatches */
for(i=0;i<nclistlength(list);i++) {
DCEprojection* p1 = (DCEprojection*)nclistget(list,i);
if(p1 == NULL) continue;
if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */
for(j=i;j<nclistlength(list);j++) {
DCEprojection* p2 = (DCEprojection*)nclistget(list,j);
if(p2 == NULL) continue;
if(p1 == p2) continue;
if(p2->discrim != CES_VAR) continue;
if(p1->var->annotation != p2->var->annotation) continue;
/* check for slice mismatches */
if(!slicematch(p1->var->segments,p2->var->segments)) {
/* complain */
nclog(NCLOGWARN,"Malformed projection: same variable with different slicing");
}
/* remove p32 */
nclistset(list,j,(ncelem)NULL);
dcefree((DCEnode*)p2);
}
}
/* Step 2: remove containers when a field is also present */
for(i=0;i<nclistlength(list);i++) {
DCEprojection* p1 = (DCEprojection*)nclistget(list,i);
if(p1 == NULL) continue;
if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */
if(!iscontainer((CDFnode*)p1->var->annotation))
continue;
for(j=i;j<nclistlength(list);j++) {
DCEprojection* p2 = (DCEprojection*)nclistget(list,j);
if(p2 == NULL) continue;
if(p2->discrim != CES_VAR) continue;
nclistclear(tmp);
collectnodepath3((CDFnode*)p2->var->annotation,tmp,WITHDATASET);
for(k=0;k<nclistlength(tmp);k++) {
void* candidate = (void*)nclistget(tmp,k);
if(candidate == p1->var->annotation) {
nclistset(list,i,(ncelem)NULL);
dcefree((DCEnode*)p1);
goto next;
}
}
}
next: continue;
}
/* Step 3: expand all containers recursively down to the leaf nodes */
for(;;) {
nclistclear(tmp);
for(i=0;i<nclistlength(list);i++) {
DCEprojection* target = (DCEprojection*)nclistget(list,i);
CDFnode* leaf;
if(target == NULL) continue;
if(target->discrim != CES_VAR)
continue; /* dont try to unify functions */
leaf = (CDFnode*)target->var->annotation;
ASSERT(leaf != NULL);
if(iscontainer(leaf)) {/* capture container */
if(!nclistcontains(tmp,(ncelem)target))
nclistpush(tmp,(ncelem)target);
nclistset(list,i,(ncelem)NULL);
}
}
if(nclistlength(tmp) == 0) break; /*done*/
/* Now explode the containers */
for(i=0;i<nclistlength(tmp);i++) {
DCEprojection* container = (DCEprojection*)nclistget(tmp,i);
CDFnode* leaf = (CDFnode*)container->var->annotation;
for(j=0;i<nclistlength(leaf->subnodes);j++) {
CDFnode* field = (CDFnode*)nclistget(leaf->subnodes,j);
/* Convert field node to a proper constraint */
DCEprojection* proj = projectify(field,container);
nclistpush(list,(ncelem)proj);
}
/* reclaim the container */
dcefree((DCEnode*)container);
}
} /*for(;;)*/
/* remove all NULL elements */
for(i=nclistlength(list)-1;i>=0;i--) {
DCEprojection* target = (DCEprojection*)nclistget(list,i);
if(target == NULL)
nclistremove(list,i);
}
done:
#ifdef DEBUG
fprintf(stderr,"fixprojection: exploded = %s\n",dumpprojections(list));
#endif
nclistfree(tmp);
return ncstat;
}
/* Compute the set of prefetched data.
Notes:
1. All prefetches are whole variable fetches.
2. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i;
NCFLAGS flags;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else {
/* pull in those variables previously marked as prefetchable */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
/* Most of the important testing was already done */
if(!var->basenode->prefetchable)
continue;
/* Do not attempt to prefetch any variables in the
nc_open url's projection list
*/
if(nclistcontains(nccomm->cdf.projectedvars,(void*)var))
continue;
/* Should be prefetchable */
nclistpush(vars,(void*)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
The exception is if we are prefetching everything.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(void*)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
flags = NCF_PREFETCH;
if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL;
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
/* Compute the set of prefetched data.
Notes:
1. Even if caching is off, we will
still prefetch the small variables.
2. All prefetches are whole variable fetches.
3. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
int isnc4 = FLAGSET(nccomm->controls,NCF_NC4);
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
if(!isnc4) {
/* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
}
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimset0);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j);
nelems *= dim->dim.declsize;
}
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems);
}
if(nelems <= nccomm->cdf.smallsizelimit) {
nclistpush(vars,(ncelem)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(ncelem)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
void
restrictprojection34(NClist* varlist, NClist* projections)
{
int i,j,len;
#ifdef DEBUG
fprintf(stderr,"restriction.before=|%s|\n",
dumpprojections(projections));
#endif
if(nclistlength(varlist) == 0) goto done; /* nothing to add or remove */
/* If the projection list is empty, then add
a projection for every variable in varlist
*/
if(nclistlength(projections) == 0) {
NClist* path = nclistnew();
NClist* nodeset = NULL;
/* Attempt to unify the vars into larger units
(like a complete grid) */
nodeset = unifyprojectionnodes3(varlist);
for(i=0;i<nclistlength(nodeset);i++) {
CDFnode* var = (CDFnode*)nclistget(nodeset,i);
#ifdef DEBUG
fprintf(stderr,"restriction.candidate=|%s|\n",var->ncfullname);
#endif
DCEprojection* newp = (DCEprojection*)dcecreate(CES_PROJECT);
newp->discrim = CES_VAR;
newp->var = (DCEvar*)dcecreate(CES_VAR);
newp->var->cdfleaf = var;
nclistclear(path);
collectnodepath3(var,path,!WITHDATASET);
newp->var->segments = nclistnew();
for(j=0;j<nclistlength(path);j++) {
CDFnode* node = (CDFnode*)nclistget(path,j);
DCEsegment* newseg = (DCEsegment*)dcecreate(CES_SEGMENT);
newseg->name = nulldup(node->name);
makewholesegment3(newseg,node);/*treat as simple projections*/
newseg->cdfnode = node;
nclistpush(newp->var->segments,(ncelem)newseg);
}
nclistpush(projections,(ncelem)newp);
}
nclistfree(path);
nclistfree(nodeset);
} else {
/* Otherwise, walk all the projections and see if they
intersect any of the variables. If not,
then remove from the projection list.
*/
len = nclistlength(projections);
for(i=len-1;i>=0;i--) {/* Walk backward to facilitate removal*/
int intersect = 0;
DCEprojection* proj = (DCEprojection*)nclistget(projections,i);
if(proj->discrim != CES_VAR) continue;
for(j=0;j<nclistlength(varlist);j++) {
CDFnode* var = (CDFnode*)nclistget(varlist,j);
/* Note that intersection could go either way */
if(treecontains3(var,proj->var->cdfleaf)
|| treecontains3(proj->var->cdfleaf,var)) {intersect = 1; break;}
}
if(!intersect) {
/* suppress this projection */
DCEprojection* p = (DCEprojection*)nclistremove(projections,i);
dcefree((DCEnode*)p);
}
}
/* Now looks for containment between projections and only keep
the more restrictive. Is this algorithm stable against reordering?.
*/
for(;;) {
int removed = 0;
for(i=0;i<nclistlength(projections);i++) {
DCEprojection* pi = (DCEprojection*)nclistget(projections,i);
if(pi->discrim != CES_VAR) continue;
for(j=0;j<i;j++) {
DCEprojection* pj = (DCEprojection*)nclistget(projections,j);
if(pj->discrim != CES_VAR) continue;
if(treecontains3(pi->var->cdfleaf,pj->var->cdfleaf)) {
DCEprojection* p = (DCEprojection*)nclistremove(projections,j);
dcefree((DCEnode*)p);
removed = 1;
break;
} else if(treecontains3(pj->var->cdfleaf,pi->var->cdfleaf)) {
DCEprojection* p = (DCEprojection*)nclistremove(projections,i);
dcefree((DCEnode*)p);
removed = 1;
break;
}
}
}
if(!removed) break;
}
}
done:
#ifdef DEBUG
fprintf(stderr,"restriction.after=|%s|\n",
dumpprojections(projections));
#endif
return;
}
