/* Compute the set of variables referenced in the projections of the input constraint. */ NCerror computeprojectedvars(NCDAPCOMMON* dapcomm, DCEconstraint* constraint) { NCerror ncstat = NC_NOERR; NClist* vars = NULL; int i; vars = nclistnew(); if(dapcomm->cdf.projectedvars != NULL) nclistfree(dapcomm->cdf.projectedvars); dapcomm->cdf.projectedvars = vars; if(constraint == NULL || constraint->projections == NULL) goto done; for(i=0;i<nclistlength(constraint->projections);i++) { CDFnode* node; DCEprojection* proj = (DCEprojection*)nclistget(constraint->projections,i); if(proj->discrim == CES_FCN) continue; /* ignore these */ node = (CDFnode*)proj->var->annotation; if(!nclistcontains(vars,(void*)node)) { nclistpush(vars,(void*)node); } } done: return ncstat; }
static void ceallnodesr(DCEnode* node, NClist* allnodes, CEsort which) { int i; if(node == NULL) return; if(nclistcontains(allnodes,(ncelem)node)) return; if(which == CES_NIL || node->sort == which) nclistpush(allnodes,(ncelem)node); switch(node->sort) { case CES_FCN: { DCEfcn* fcn = (DCEfcn*)node; for(i=0;i<nclistlength(fcn->args);i++) { ceallnodesr((DCEnode*)nclistget(fcn->args,i),allnodes,which); } } break; case CES_VAR: { DCEvar* var = (DCEvar*)node; for(i=0;i<nclistlength(var->segments);i++) { ceallnodesr((DCEnode*)nclistget(var->segments,i),allnodes,which); } } break; case CES_VALUE: { DCEvalue* value = (DCEvalue*)node; if(value->discrim == CES_VAR) ceallnodesr((DCEnode*)value->var,allnodes,which); else if(value->discrim == CES_FCN) ceallnodesr((DCEnode*)value->fcn,allnodes,which); else ceallnodesr((DCEnode*)value->constant,allnodes,which); } break; case CES_SELECT: { DCEselection* selection = (DCEselection*)node; ceallnodesr((DCEnode*)selection->lhs,allnodes,which); for(i=0;i<nclistlength(selection->rhs);i++) ceallnodesr((DCEnode*)nclistget(selection->rhs,i),allnodes,which); } break; case CES_PROJECT: { DCEprojection* projection = (DCEprojection*)node; if(projection->discrim == CES_VAR) ceallnodesr((DCEnode*)projection->var,allnodes,which); else ceallnodesr((DCEnode*)projection->fcn,allnodes,which); } break; case CES_CONSTRAINT: { DCEconstraint* constraint = (DCEconstraint*)node; for(i=0;i<nclistlength(constraint->projections);i++) ceallnodesr((DCEnode*)nclistget(constraint->projections,i),allnodes,which); for(i=0;i<nclistlength(constraint->selections);i++) ceallnodesr((DCEnode*)nclistget(constraint->selections,i),allnodes,which); } break; /* All others have no subnodes */ default: break; } }
static NCerror sequencecheckr(CDFnode* node, NClist* vars, CDFnode* topseq) { unsigned int i; NCerror err = NC_NOERR; int ok = 0; if(topseq == NULL && nclistlength(node->array.dimset0) > 0) { err = NC_EINVAL; /* This container has dimensions, so no sequence within it can be usable */ } else if(node->nctype == NC_Sequence) { /* Recursively walk the path for each subnode of this sequence node looking for a path without any sequence */ for(i=0; i<nclistlength(node->subnodes); i++) { CDFnode* sub = (CDFnode*)nclistget(node->subnodes,i); err = sequencecheckr(sub,vars,node); if(err == NC_NOERR) ok = 1; /* there is at least 1 usable var below */ } if(topseq == NULL && ok == 1) { /* this sequence is usable because it has scalar container (by construction) and has a path to a leaf without an intermediate sequence. */ err = NC_NOERR; node->usesequence = 1; } else { /* this sequence is unusable because it has no path to a leaf without an intermediate sequence. */ node->usesequence = 0; err = NC_EINVAL; } } else if(nclistcontains(vars,(void*)node)) { /* If we reach a leaf, then topseq is usable, so save it */ node->array.sequence = topseq; } else { /* Some kind of non-sequence container node with no dimensions */ /* recursively compute usability */ for(i=0; i<nclistlength(node->subnodes); i++) { CDFnode* sub = (CDFnode*)nclistget(node->subnodes,i); err = sequencecheckr(sub,vars,topseq); if(err == NC_NOERR) ok = 1; } err = (ok?NC_NOERR:NC_EINVAL); } return err; }
NCerror fixprojections(NClist* list) { int i,j,k; NCerror ncstat = NC_NOERR; NClist* tmp = nclistnew(); /* misc. uses */ #ifdef DEBUG fprintf(stderr,"fixprojection: list = %s\n",dumpprojections(list)); #endif if(nclistlength(list) == 0) goto done; /* Step 1: remove duplicates and complain about slice mismatches */ for(i=0;i<nclistlength(list);i++) { DCEprojection* p1 = (DCEprojection*)nclistget(list,i); if(p1 == NULL) continue; if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */ for(j=i;j<nclistlength(list);j++) { DCEprojection* p2 = (DCEprojection*)nclistget(list,j); if(p2 == NULL) continue; if(p1 == p2) continue; if(p2->discrim != CES_VAR) continue; if(p1->var->annotation != p2->var->annotation) continue; /* check for slice mismatches */ if(!slicematch(p1->var->segments,p2->var->segments)) { /* complain */ nclog(NCLOGWARN,"Malformed projection: same variable with different slicing"); } /* remove p32 */ nclistset(list,j,(ncelem)NULL); dcefree((DCEnode*)p2); } } /* Step 2: remove containers when a field is also present */ for(i=0;i<nclistlength(list);i++) { DCEprojection* p1 = (DCEprojection*)nclistget(list,i); if(p1 == NULL) continue; if(p1->discrim != CES_VAR) continue; /* dont try to unify functions */ if(!iscontainer((CDFnode*)p1->var->annotation)) continue; for(j=i;j<nclistlength(list);j++) { DCEprojection* p2 = (DCEprojection*)nclistget(list,j); if(p2 == NULL) continue; if(p2->discrim != CES_VAR) continue; nclistclear(tmp); collectnodepath3((CDFnode*)p2->var->annotation,tmp,WITHDATASET); for(k=0;k<nclistlength(tmp);k++) { void* candidate = (void*)nclistget(tmp,k); if(candidate == p1->var->annotation) { nclistset(list,i,(ncelem)NULL); dcefree((DCEnode*)p1); goto next; } } } next: continue; } /* Step 3: expand all containers recursively down to the leaf nodes */ for(;;) { nclistclear(tmp); for(i=0;i<nclistlength(list);i++) { DCEprojection* target = (DCEprojection*)nclistget(list,i); CDFnode* leaf; if(target == NULL) continue; if(target->discrim != CES_VAR) continue; /* dont try to unify functions */ leaf = (CDFnode*)target->var->annotation; ASSERT(leaf != NULL); if(iscontainer(leaf)) {/* capture container */ if(!nclistcontains(tmp,(ncelem)target)) nclistpush(tmp,(ncelem)target); nclistset(list,i,(ncelem)NULL); } } if(nclistlength(tmp) == 0) break; /*done*/ /* Now explode the containers */ for(i=0;i<nclistlength(tmp);i++) { DCEprojection* container = (DCEprojection*)nclistget(tmp,i); CDFnode* leaf = (CDFnode*)container->var->annotation; for(j=0;i<nclistlength(leaf->subnodes);j++) { CDFnode* field = (CDFnode*)nclistget(leaf->subnodes,j); /* Convert field node to a proper constraint */ DCEprojection* proj = projectify(field,container); nclistpush(list,(ncelem)proj); } /* reclaim the container */ dcefree((DCEnode*)container); } } /*for(;;)*/ /* remove all NULL elements */ for(i=nclistlength(list)-1;i>=0;i--) { DCEprojection* target = (DCEprojection*)nclistget(list,i); if(target == NULL) nclistremove(list,i); } done: #ifdef DEBUG fprintf(stderr,"fixprojection: exploded = %s\n",dumpprojections(list)); #endif nclistfree(tmp); return ncstat; }
/* Compute the set of prefetched data. Notes: 1. All prefetches are whole variable fetches. 2. If the data set is unconstrainable, we will prefetch the whole thing */ NCerror prefetchdata3(NCDAPCOMMON* nccomm) { int i; NCFLAGS flags; NCerror ncstat = NC_NOERR; NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes; DCEconstraint* urlconstraint = nccomm->oc.dapconstraint; NClist* vars = nclistnew(); NCcachenode* cache = NULL; DCEconstraint* newconstraint = NULL; if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) { /* If we cannot constrain and caching is enabled, then pull in everything */ if(FLAGSET(nccomm->controls,NCF_CACHE)) { for(i=0;i<nclistlength(allvars);i++) { nclistpush(vars,nclistget(allvars,i)); } } else { /* do no prefetching */ nccomm->cdf.cache->prefetch = NULL; goto done; } } else { /* pull in those variables previously marked as prefetchable */ for(i=0;i<nclistlength(allvars);i++) { CDFnode* var = (CDFnode*)nclistget(allvars,i); /* Most of the important testing was already done */ if(!var->basenode->prefetchable) continue; /* Do not attempt to prefetch any variables in the nc_open url's projection list */ if(nclistcontains(nccomm->cdf.projectedvars,(void*)var)) continue; /* Should be prefetchable */ nclistpush(vars,(void*)var); if(SHOWFETCH) { nclog(NCLOGDBG,"prefetch: %s",var->ncfullname); } } } /* If there are no vars, then do nothing */ if(nclistlength(vars) == 0) { nccomm->cdf.cache->prefetch = NULL; goto done; } /* Create a single constraint consisting of the projections for the variables; each projection is whole variable. The selections are passed on as is. The exception is if we are prefetching everything. */ newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT); newconstraint->projections = nclistnew(); newconstraint->selections = dceclonelist(urlconstraint->selections); for(i=0;i<nclistlength(vars);i++) { CDFnode* var = (CDFnode*)nclistget(vars,i); DCEprojection* varprojection; /* convert var to a projection */ ncstat = dapvar2projection(var,&varprojection); if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} nclistpush(newconstraint->projections,(void*)varprojection); } if(SHOWFETCH) { char* s = dumpprojections(newconstraint->projections); LOG1(NCLOGNOTE,"prefetch.final: %s",s); nullfree(s); } flags = NCF_PREFETCH; if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL; ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags); newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */ if(ncstat) goto done; cache->wholevariable = 1; /* All prefetches are whole variable */ /* Make cache node be the prefetch node */ nccomm->cdf.cache->prefetch = cache; if(SHOWFETCH) { LOG0(NCLOGNOTE,"prefetch.complete"); } if(SHOWFETCH) { char* s = NULL; /* Log the set of prefetch variables */ NCbytes* buf = ncbytesnew(); ncbytescat(buf,"prefetch.vars: "); for(i=0;i<nclistlength(vars);i++) { CDFnode* var = (CDFnode*)nclistget(vars,i); ncbytescat(buf," "); s = makecdfpathstring3(var,"."); ncbytescat(buf,s); nullfree(s); } ncbytescat(buf,"\n"); nclog(NCLOGNOTE,"%s",ncbytescontents(buf)); ncbytesfree(buf); } done: nclistfree(vars); dcefree((DCEnode*)newconstraint); if(ncstat) freenccachenode(nccomm,cache); return THROW(ncstat); }
/* See if we can unify sets of nodes to be projected into larger units. */ static NClist* unifyprojectionnodes3(NClist* varlist) { int i; NClist* nodeset = nclistnew(); NClist* containerset = nclistnew(); NClist* containernodes = nclistnew(); nclistsetalloc(nodeset,nclistlength(varlist)); nclistsetalloc(containerset,nclistlength(varlist)); /* Duplicate the varlist so we can modify it; simultaneously collect unique container set. */ for(i=0;i<nclistlength(varlist);i++) { CDFnode* var = (CDFnode*)nclistget(varlist,i); CDFnode* container = var->container; nclistpush(nodeset,(ncelem)var); switch (container->nctype) { case NC_Sequence: case NC_Structure: case NC_Grid: case NC_Dataset: /* add (uniquely) to container set */ if(!nclistcontains(containerset,(ncelem)container)) nclistpush(containerset,(ncelem)container); break; default: break; } } /* Now, try to find containers whose subnodes are all in the varlist; repeat until no more changes */ for(;;) { int changed = 0; for(i=0;i<nclistlength(containerset);i++) { int j, allfound; CDFnode* container = (CDFnode*)nclistget(containerset,i); if(container == NULL) continue; nclistclear(containernodes); for(allfound=1,j=0;j<nclistlength(container->subnodes);j++) { CDFnode* subnode = (CDFnode*)nclistget(container->subnodes,j); if(!nclistcontains(varlist,(ncelem)subnode)) {allfound=0;break;} nclistpush(containernodes,(ncelem)subnode); } if(allfound) { nclistpush(nodeset,(ncelem)container); nclistset(containerset,i,(ncelem)NULL); /* remove */ for(j=nclistlength(nodeset)-1;j>=0;j--) { /* walk backwards */ CDFnode* testnode = (CDFnode*)nclistget(nodeset,j); if(nclistcontains(containernodes,(ncelem)testnode)) nclistremove(nodeset,j);/* remove */ } changed = 1; } } if(!changed) break; /* apparently we have reached a stable situation */ } /* If there is only the dataset left as a projection, then remove it */ if(nclistlength(nodeset) == 1) { CDFnode* thenode = (CDFnode*)nclistget(nodeset,0); if(thenode->nctype == NC_Dataset) nclistclear(nodeset); } nclistfree(containerset); nclistfree(containernodes); return nodeset; }