/* Estimate variables sizes and then resort the variable list
by that size
*/
void
estimatevarsizes3(NCDAPCOMMON* dapcomm)
{
int ivar;
unsigned int rank;
size_t totalsize = 0;
for(ivar=0;ivar<nclistlength(dapcomm->cdf.ddsroot->tree->varnodes);ivar++) {
CDFnode* var = (CDFnode*)nclistget(dapcomm->cdf.ddsroot->tree->varnodes,ivar);
NClist* ncdims = var->array.dimset0;
rank = nclistlength(ncdims);
if(rank == 0) { /* use instance size of the type */
var->estimatedsize = nctypesizeof(var->etype);
#ifdef DEBUG1
fprintf(stderr,"scalar %s.estimatedsize = %lu\n",
makecdfpathstring3(var,"."),var->estimatedsize);
#endif
} else {
unsigned long size = cdftotalsize3(ncdims);
size *= nctypesizeof(var->etype);
#ifdef DEBUG1
fprintf(stderr,"array %s(%u).estimatedsize = %lu\n",
makecdfpathstring3(var,"."),rank,size);
#endif
var->estimatedsize = size;
}
totalsize += var->estimatedsize;
}
#ifdef DEBUG1
fprintf(stderr,"total estimatedsize = %lu\n",totalsize);
#endif
dapcomm->cdf.totalestimatedsize = totalsize;
}
char*
dumpselection1(NCselection* sel)
{
NCbytes* buf = ncbytesnew();
char* sstring;
NClist* path = NULL;
char* pathstring = NULL;
int j;
if(sel == NULL) return nulldup("");
path = sel->path;
ncbytescat(buf,"&");
if(path == NULL)
pathstring = makecdfpathstring3(sel->node,".");
else
pathstring = simplepathstring3(path,".");
ncbytescat(buf,pathstring);
efree(pathstring);
ncbytescat(buf,opstrings[sel->operator]);
ncbytescat(buf,"{");
for(j=0;j<nclistlength(sel->values);j++) {
NCvalue* value = (NCvalue*)nclistget(sel->values,j);
char tmp[64];
if(j > 0) ncbytescat(buf,",");
switch (value->kind) {
case ST_STR:
ncbytescat(buf,value->value.text);
break;
case ST_INT:
snprintf(tmp,sizeof(tmp),"%lld",value->value.intvalue);
ncbytescat(buf,tmp);
break;
case ST_FLOAT:
snprintf(tmp,sizeof(tmp),"%g",value->value.floatvalue);
ncbytescat(buf,tmp);
break;
case ST_VAR:
path = value->value.var.path;
if(path == NULL)
pathstring = makecdfpathstring3(value->value.var.node,".");
else
pathstring = simplepathstring3(path,".");
ncbytescat(buf,pathstring);
efree(pathstring);
break;
default: PANIC1("unexpected tag: %d",(int)value->kind);
}
}
ncbytescat(buf,"}");
sstring = ncbytesdup(buf);
ncbytesfree(buf);
return sstring;
}
char*
dumpcachenode(NCcachenode* node)
{
char* result = NULL;
char tmp[8192];
int i;
NCbytes* buf;
if(node == NULL) return strdup("cachenode{null}");
buf = ncbytesnew();
result = buildconstraintstring3(node->constraint);
snprintf(tmp,sizeof(tmp),"cachenode%s(%lx){size=%lu; constraint=%s; vars=",
node->isprefetch?"*":"",
(unsigned long)node,
(unsigned long)node->xdrsize,
result);
ncbytescat(buf,tmp);
if(nclistlength(node->vars)==0)
ncbytescat(buf,"null");
else for(i=0;i<nclistlength(node->vars);i++) {
CDFnode* var = (CDFnode*)nclistget(node->vars,i);
if(i > 0) ncbytescat(buf,",");
ncbytescat(buf,makecdfpathstring3(var,"."));
}
ncbytescat(buf,"}");
result = ncbytesdup(buf);
ncbytesfree(buf);
return result;
}
/* Compute the set of prefetched data.
Notes:
1. All prefetches are whole variable fetches.
2. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i;
NCFLAGS flags;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else {
/* pull in those variables previously marked as prefetchable */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
/* Most of the important testing was already done */
if(!var->basenode->prefetchable)
continue;
/* Do not attempt to prefetch any variables in the
nc_open url's projection list
*/
if(nclistcontains(nccomm->cdf.projectedvars,(void*)var))
continue;
/* Should be prefetchable */
nclistpush(vars,(void*)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
The exception is if we are prefetching everything.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(void*)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
flags = NCF_PREFETCH;
if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL;
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
/* Return 1 if we can reuse cached data to address
the current get_vara request; return 0 otherwise.
Target is in the constrained tree space.
Currently, if the target matches a cache that is not
a whole variable, then match is false.
*/
int
iscached(NCDAPCOMMON* nccomm, CDFnode* target, NCcachenode** cachenodep)
{
int i,j,found,index;
NCcache* cache;
NCcachenode* cachenode;
found = 0;
if(target == NULL) goto done;
/* Match the target variable against the prefetch, if any */
/* Note that prefetches are always whole variable */
cache = nccomm->cdf.cache;
cachenode = cache->prefetch;
if(cachenode!= NULL) {
for(found=0,i=0;i<nclistlength(cachenode->vars);i++) {
CDFnode* var = (CDFnode*)nclistget(cachenode->vars,i);
if(var == target) {
if(cachenodep) *cachenodep = cachenode;
found=1;
goto done;
}
}
}
/*search other cache nodes starting at latest first */
index = 0;
for(i=nclistlength(cache->nodes)-1;i>=0;i--) {
cachenode = (NCcachenode*)nclistget(cache->nodes,i);
/* We currently do not try to match constraints;
If the cachenode is constrained by more than
simple wholevariable projections, then skip it.
*/
if(!cachenode->wholevariable) continue;
for(found=0,j=0;j<nclistlength(cachenode->vars);j++) {
CDFnode* var = (CDFnode*)nclistget(cachenode->vars,j);
if(var == target) {found=1;index=i;break;}
}
if(found) break;
}
if(found) {
ASSERT((cachenode != NULL));
if(nclistlength(cache->nodes) > 1) {
/* Manage the cache nodes as LRU */
nclistremove(cache->nodes,index);
nclistpush(cache->nodes,(void*)cachenode);
}
if(cachenodep) *cachenodep = cachenode;
}
done:
#ifdef DEBUG
fprintf(stderr,"iscached: search: %s\n",makecdfpathstring3(target,"."));
if(found)
fprintf(stderr,"iscached: found: %s\n",dumpcachenode(cachenode));
else
fprintf(stderr,"iscached: notfound\n");
#endif
return found;
}
NCerror
computevarnodes3(NCDAPCOMMON* nccomm, NClist* allnodes, NClist* varnodes)
{
unsigned int i,len;
NClist* allvarnodes = nclistnew();
for(i=0;i<nclistlength(allnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(allnodes,i);
#if 0
/* If this node has a bad name, repair it */
if(dap_badname(node->ocname)) {
char* newname = dap_repairname(node->ocname);
nullfree(node->ocname);
node->ocname = newname;
}
#endif
if(node->nctype == NC_Atomic)
nclistpush(allvarnodes,(void*)node);
}
/* Further process the variable nodes to get the final set */
/* Use toplevel vars first */
len = nclistlength(allvarnodes);
for(i=0;i<len;i++) {
CDFnode* node = (CDFnode*)nclistget(allvarnodes,i);
if(node == NULL) continue;
if(daptoplevel(node)) {
nclistpush(varnodes,(void*)node);
nclistset(allvarnodes,i,(void*)NULL);
}
}
/*... then grid arrays and maps.
but exclude the coordinate variables if we are trying to
exactly mimic nc-dap
*/
for(i=0;i<len;i++) {
CDFnode* node = (CDFnode*)nclistget(allvarnodes,i);
if(node == NULL) continue;
if(dapgridarray(node)) {
nclistpush(varnodes,(void*)node);
nclistset(allvarnodes,i,(void*)NULL);
} else if(dapgridmap(node)) {
if(!FLAGSET(nccomm->controls,NCF_NCDAP))
nclistpush(varnodes,(void*)node);
nclistset(allvarnodes,i,(void*)NULL);
}
}
/*... then all others */
for(i=0;i<len;i++) {
CDFnode* node = (CDFnode*)nclistget(allvarnodes,i);
if(node == NULL) continue;
nclistpush(varnodes,(void*)node);
}
nclistfree(allvarnodes);
#ifdef DEBUG2
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(varnodes,i);
if(node == NULL) continue;
fprintf(stderr,"computevarnodes: var: %s\n",makecdfpathstring3(node,"."));
}
#endif
return NC_NOERR;
}
/*
Figure out the names for variables.
*/
NCerror
computecdfvarnames3(NCDAPCOMMON* nccomm, CDFnode* root, NClist* varnodes)
{
unsigned int i,j,d;
/* clear all elided marks; except for dataset and grids */
for(i=0;i<nclistlength(root->tree->nodes);i++) {
CDFnode* node = (CDFnode*)nclistget(root->tree->nodes,i);
node->elided = 0;
if(node->nctype == NC_Grid || node->nctype == NC_Dataset)
node->elided = 1;
}
/* ensure all variables have an initial full name defined */
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* var = (CDFnode*)nclistget(varnodes,i);
nullfree(var->ncfullname);
var->ncfullname = makecdfpathstring3(var,nccomm->cdf.separator);
#ifdef DEBUG2
fprintf(stderr,"var names: %s %s %s\n",
var->ocname,var->ncbasename,var->ncfullname);
#endif
}
/* unify all variables with same fullname and dimensions
basevar fields says: "for duplicate grid variables";
when does this happen?
*/
if(FLAGSET(nccomm->controls,NCF_NC3)) {
for(i=0;i<nclistlength(varnodes);i++) {
int match;
CDFnode* var = (CDFnode*)nclistget(varnodes,i);
for(j=0;j<i;j++) {
CDFnode* testnode = (CDFnode*)nclistget(varnodes,j);
match = 1;
if(testnode->array.basevar != NULL)
continue; /* already processed */
if(strcmp(var->ncfullname,testnode->ncfullname) != 0)
match = 0;
else if(nclistlength(testnode->array.dimsetall)
!= nclistlength(var->array.dimsetall))
match = 0;
else for(d=0;d<nclistlength(testnode->array.dimsetall);d++) {
CDFnode* vdim = (CDFnode*)nclistget(var->array.dimsetall,d);
CDFnode* tdim = (CDFnode*)nclistget(testnode->array.dimsetall,d);
if(vdim->dim.declsize != tdim->dim.declsize) {
match = 0;
break;
}
}
if(match) {
testnode->array.basevar = var;
fprintf(stderr,"basevar invoked: %s\n",var->ncfullname);
}
}
}
}
/* Finally, verify unique names */
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* var1 = (CDFnode*)nclistget(varnodes,i);
if(var1->array.basevar != NULL) continue;
for(j=0;j<i;j++) {
CDFnode* var2 = (CDFnode*)nclistget(varnodes,j);
if(var2->array.basevar != NULL) continue;
if(strcmp(var1->ncfullname,var2->ncfullname)==0) {
PANIC1("duplicate var names: %s",var1->ncfullname);
}
}
}
return NC_NOERR;
}
int
dapmerge3(NCDAPCOMMON* nccomm, CDFnode* ddsroot, OCobject dasroot)
{
unsigned int i,j;
NCerror ncerr = NC_NOERR;
OCerror ocstat = OC_NOERR;
OCconnection conn = nccomm->oc.conn;
unsigned int nsubnodes, nobjects;
OCobject* dasobjects = NULL;
NClist* dasglobals = nclistnew();
NClist* dasnodes = nclistnew();
NClist* dodsextra = nclistnew();
NClist* varnodes = nclistnew();
NClist* allddsnodes = ddsroot->tree->nodes;
if(ddsroot == NULL || dasroot == NULL) return NC_NOERR;
nobjects = oc_inq_nobjects(conn,dasroot);
dasobjects = oc_inq_objects(conn,dasroot);
/* 1. collect all the relevant DAS nodes;
namely those that contain at least one
attribute value.
Simultaneously look for potential ambiguities
if found; complain but continue: result are indeterminate.
also collect globals and DODS_EXTRA separately.
*/
for(i=0;i<nobjects;i++) {
OCobject das = dasobjects[i];
OCtype octype;
char* ocname = NULL;
int isglobal = 0;
int hasattributes = 0;
OCHECK(oc_inq_class(conn,das,&octype));
if(octype == OC_Attribute) continue; /* ignore these for now*/
OCHECK(oc_inq_name(conn,das,&ocname));
OCHECK(oc_inq_nsubnodes(conn,das,&nsubnodes));
isglobal = (ocname == NULL ? 0 : isglobalname3(ocname));
/* catch DODS_EXTRA */
if(isglobal && ocname != NULL && strcmp(ocname,"DODS_EXTRA")==0) {
nclistpush(dodsextra,(ncelem)das);
nullfree(ocname);
continue;
}
if(ocname == NULL || isglobal) {
nclistpush(dasglobals,(ncelem)das);
nullfree(ocname);
continue;
}
hasattributes = hasattribute3(conn,das);
if(hasattributes) {
/* Look for previously collected nodes with same name*/
for(j=0;j<nclistlength(dasnodes);j++) {
OCobject das2 = (OCobject)nclistget(dasnodes,j);
char* ocname2;
OCHECK(oc_inq_name(conn,das2,&ocname2));
if(ocname2 == NULL || ocname == NULL) goto loop;
if(strcmp(ocname2,"DODS")==0) goto loop;
if(strcmp(ocname,ocname2)==0)
nclog(NCLOGWARN,"nc_mergedas: potentially ambiguous DAS name: %s",ocname2);
loop:
nullfree(ocname2);
}
nclistpush(dasnodes,(ncelem)das);
}
nullfree(ocname);
}
/* 2. collect all the leaf DDS nodes (of type NC_Primitive)*/
for(i=0;i<nclistlength(allddsnodes);i++) {
CDFnode* dds = (CDFnode*)nclistget(allddsnodes,i);
if(dds->nctype == NC_Primitive) nclistpush(varnodes,(ncelem)dds);
}
/* 3. For each das node, lncate matching DDS node(s) and attach
attributes to the DDS node(s).
Match means:
1. DAS->fullname :: DDS->fullname
2. DAS->name :: DDS->fullname (support DAS names with embedded '.'
3. DAS->name :: DDS->name
4. special case for DODS. Apply 1-3 on DODS parent.
*/
for(i=0;i<nclistlength(dasnodes);i++) {
OCobject das = (OCobject)nclistget(dasnodes,i);
char* ocfullname = NULL;
char* ocbasename = NULL;
if(das == OCNULL) continue;
OCHECK(oc_inq_name(conn,das,&ocbasename));
if(strcmp(ocbasename,"DODS")==0) {
OCobject container;
OCHECK(oc_inq_container(conn,das,&container));
if(container == OCNULL) {
ASSERT(container != OCNULL);
}
ocfullname = makeocpathstring3(conn,container,".");
} else {
ocfullname = makeocpathstring3(conn,das,".");
}
for(j=0;j<nclistlength(varnodes);j++) {
CDFnode* dds = (CDFnode*)nclistget(varnodes,j);
char* ddsfullname = makecdfpathstring3(dds,".");
if(strcmp(ocfullname,ddsfullname)==0
|| strcmp(ocbasename,ddsfullname)==0
|| strcmp(ocbasename,dds->ocname)==0) {
mergedas1(nccomm,conn,dds,das);
/* remove from dasnodes list*/
nclistset(dasnodes,i,(ncelem)NULL);
}
nullfree(ddsfullname);
}
nullfree(ocfullname);
nullfree(ocbasename);
}
/* 4. Assign globals */
for(i=0;i<nclistlength(dasglobals);i++) {
OCobject das = (OCobject)nclistget(dasglobals,i);
mergedas1(nccomm,conn,ddsroot,das);
}
/* 5. Assign DOD_EXTRA */
for(i=0;i<nclistlength(dodsextra);i++) {
OCobject das = (OCobject)nclistget(dodsextra,i);
mergedas1(nccomm,conn,ddsroot,das);
}
done: /* cleanup*/
nullfree(dasobjects);
nclistfree(dasglobals);
nclistfree(dasnodes);
nclistfree(dodsextra);
nclistfree(varnodes);
if(ocstat != OC_NOERR) ncerr = ocerrtoncerr(ocstat);
return THROW(ncerr);
}
/* Compute the set of prefetched data.
Notes:
1. Even if caching is off, we will
still prefetch the small variables.
2. All prefetches are whole variable fetches.
3. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
int isnc4 = FLAGSET(nccomm->controls,NCF_NC4);
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
if(!isnc4) {
/* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
}
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimset0);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j);
nelems *= dim->dim.declsize;
}
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems);
}
if(nelems <= nccomm->cdf.smallsizelimit) {
nclistpush(vars,(ncelem)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(ncelem)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
