char*
dumpcachenode(NCcachenode* node)
{
char* result = NULL;
char tmp[8192];
int i;
NCbytes* buf;
if(node == NULL) return strdup("cachenode{null}");
buf = ncbytesnew();
snprintf(tmp,sizeof(tmp),"cachenode%s(%lx){size=%lu; constraint=%s; vars=",
node->prefetch?"*":"",
(unsigned long)node,
(unsigned long)node->xdrsize,
buildconstraintstring3(node->constraint));
ncbytescat(buf,tmp);
if(nclistlength(node->vars)==0)
ncbytescat(buf,"null");
else for(i=0;i<nclistlength(node->vars);i++) {
CDFnode* var = (CDFnode*)nclistget(node->vars,i);
if(i > 0) ncbytescat(buf,",");
ncbytescat(buf,makesimplepathstring3(var));
}
ncbytescat(buf,"}");
result = ncbytesdup(buf);
ncbytesfree(buf);
return result;
}
NCerror
fetchconstrainedmetadata3(NCDAPCOMMON* dapcomm)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
OCddsnode ocroot;
CDFnode* ddsroot; /* constrained */
char* ce = NULL;
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE))
ce = NULL;
else
ce = buildconstraintstring3(dapcomm->oc.dapconstraint);
{
ocstat = dap_fetch(dapcomm,dapcomm->oc.conn,ce,OCDDS,&ocroot);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;}
/* Construct our parallel dds tree; including attributes*/
ncstat = buildcdftree34(dapcomm,ocroot,OCDDS,&ddsroot);
if(ncstat) goto fail;
ocroot = NULL; /* avoid duplicate reclaim */
dapcomm->cdf.ddsroot = ddsroot;
ddsroot = NULL; /* to avoid double reclamation */
if(!FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
/* fix DAP server problem by adding back any inserting needed structure nodes */
ncstat = restruct3(dapcomm->cdf.ddsroot,dapcomm->cdf.fullddsroot,dapcomm->oc.dapconstraint->projections);
if(ncstat) goto fail;
}
#ifdef DEBUG
fprintf(stderr,"constrained:\n%s",dumptree(dapcomm->cdf.ddsroot));
#endif
/* Combine DDS and DAS */
if(dapcomm->oc.ocdasroot != NULL) {
ncstat = dapmerge3(dapcomm,dapcomm->cdf.ddsroot->ocnode,
dapcomm->oc.ocdasroot);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
}
/* map the constrained DDS to the unconstrained DDS */
ncstat = mapnodes3(dapcomm->cdf.ddsroot,dapcomm->cdf.fullddsroot);
if(ncstat) goto fail;
}
fail:
nullfree(ce);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return ncstat;
}
/* See ncd3dispatch.c for other version */
int
NCD3_open(const char * path, int mode,
int basepe, size_t *chunksizehintp,
int useparallel, void* mpidata,
NC_Dispatch* dispatch, NC** ncpp)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
NC* drno = NULL;
NCDAPCOMMON* dapcomm = NULL;
const char* value;
char* tmpname = NULL;
if(!nc3dinitialized) nc3dinitialize();
if(path == NULL)
return NC_EDAPURL;
if(dispatch == NULL) PANIC("NC3D_open: no dispatch table");
/* Setup our NC and NCDAPCOMMON state*/
drno = (NC*)calloc(1,sizeof(NC));
if(drno == NULL) {ncstat = NC_ENOMEM; goto done;}
/* compute an ncid */
ncstat = add_to_NCList(drno);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
dapcomm = (NCDAPCOMMON*)calloc(1,sizeof(NCDAPCOMMON));
if(dapcomm == NULL) {ncstat = NC_ENOMEM; goto done;}
drno->dispatch = dispatch;
drno->dispatchdata = dapcomm;
dapcomm->controller = (NC*)drno;
dapcomm->cdf.separator = ".";
dapcomm->cdf.smallsizelimit = DFALTSMALLLIMIT;
dapcomm->cdf.cache = createnccache();
#ifdef HAVE_GETRLIMIT
{ struct rlimit rl;
if(getrlimit(RLIMIT_NOFILE, &rl) >= 0) {
dapcomm->cdf.cache->cachecount = (size_t)(rl.rlim_cur / 2);
}
}
#endif
#ifdef OCCOMPILEBYDEFAULT
/* set the compile flag by default */
dapcomm->oc.rawurltext = (char*)emalloc(strlen(path)+strlen("[compile]")+1);
strcpy(dapcomm->oc.rawurltext,"[compile]");
strcat(dapcomm->oc.rawurltext, path);
#else
dapcomm->oc.rawurltext = strdup(path);
#endif
nc_uriparse(dapcomm->oc.rawurltext,&dapcomm->oc.url);
/* parse the client parameters */
nc_uridecodeparams(dapcomm->oc.url);
if(!constrainable34(dapcomm->oc.url))
SETFLAG(dapcomm->controls,NCF_UNCONSTRAINABLE);
/* Use libsrc code for storing metadata */
tmpname = nulldup(PSEUDOFILE);
/* Now, use the file to create the netcdf file */
if(sizeof(size_t) == sizeof(unsigned int))
ncstat = nc_create(tmpname,NC_CLOBBER,&drno->substrate);
else
ncstat = nc_create(tmpname,NC_CLOBBER|NC_64BIT_OFFSET,&drno->substrate);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* free the filename so it will automatically go away*/
unlink(tmpname);
nullfree(tmpname);
/* Avoid fill */
nc_set_fill(drno->substrate,NC_NOFILL,NULL);
dapcomm->oc.dapconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
dapcomm->oc.dapconstraint->projections = nclistnew();
dapcomm->oc.dapconstraint->selections = nclistnew();
/* Parse constraints to make sure they are syntactically correct */
ncstat = parsedapconstraints(dapcomm,dapcomm->oc.url->constraint,dapcomm->oc.dapconstraint);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Complain if we are unconstrainable but have constraints */
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
if(dapcomm->oc.url->constraint != NULL
&& strlen(dapcomm->oc.url->constraint) > 0) {
nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s",
dapcomm->oc.url->constraint);
}
}
/* Construct a url for oc minus any parameters */
dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,
(NC_URIALL ^ NC_URICONSTRAINTS));
/* Pass to OC */
ocstat = oc_open(dapcomm->oc.urltext,&dapcomm->oc.conn);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
nullfree(dapcomm->oc.urltext); /* clean up */
dapcomm->oc.urltext = NULL;
/* process control client parameters */
applyclientparamcontrols3(dapcomm);
/* Turn on logging; only do this after oc_open*/
if((value = paramvalue34(dapcomm,"log")) != NULL) {
ncloginit();
ncsetlogging(1);
nclogopen(value);
oc_loginit();
oc_setlogging(1);
oc_logopen(value);
}
/* fetch and build the (almost) unconstrained DDS for use as
template */
ncstat = fetchtemplatemetadata3(dapcomm);
if(ncstat != NC_NOERR) goto done;
/* fetch and build the constrained DDS */
ncstat = fetchconstrainedmetadata3(dapcomm);
if(ncstat != NC_NOERR) goto done;
#ifdef DEBUG2
fprintf(stderr,"constrained dds: %s\n",dumptree(dapcomm->cdf.ddsroot));
#endif
/* The following actions are (mostly) WRT to the constrained tree */
/* Accumulate useful nodes sets */
ncstat = computecdfnodesets3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Fix grids */
ncstat = fixgrids3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Locate and mark usable sequences */
ncstat = sequencecheck3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* suppress variables not in usable sequences */
ncstat = suppressunusablevars3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* apply client parameters */
ncstat = applyclientparams34(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Add (as needed) string dimensions*/
ncstat = addstringdims(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
if(nclistlength(dapcomm->cdf.seqnodes) > 0) {
/* Build the sequence related dimensions */
ncstat = defseqdims(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
}
/* Define the dimsetplus and dimsetall lists */
ncstat = definedimsets3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Re-compute the dimension names*/
ncstat = computecdfdimnames34(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Deal with zero size dimensions */
ncstat = fixzerodims3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Attempt to use the DODS_EXTRA info to turn
one of the dimensions into unlimited.
Assume computecdfdimnames34 has already been called.
*/
ncstat = defrecorddim3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
if(dapcomm->cdf.recorddimname != NULL
&& nclistlength(dapcomm->cdf.seqnodes) > 0) {
/*nclog(NCLOGWARN,"unlimited dimension specified, but sequences exist in DDS");*/
PANIC("unlimited dimension specified, but sequences exist in DDS");
}
/* Re-compute the var names*/
ncstat = computecdfvarnames3(dapcomm,dapcomm->cdf.ddsroot,dapcomm->cdf.varnodes);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Transfer data from the unconstrained DDS data to the unconstrained DDS */
ncstat = dimimprint3(dapcomm);
if(ncstat) goto done;
/* Process the constraints to map to the constrained CDF tree */
/* (must follow fixgrids3 */
ncstat = mapconstraints3(dapcomm->oc.dapconstraint,dapcomm->cdf.ddsroot);
if(ncstat != NC_NOERR) goto done;
/* Canonicalize the constraint */
ncstat = fixprojections(dapcomm->oc.dapconstraint->projections);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Fill in segment information */
ncstat = qualifyconstraints3(dapcomm->oc.dapconstraint);
if(ncstat != NC_NOERR) goto done;
/* using the modified constraint, rebuild the constraint string */
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
/* ignore all constraints */
dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,0);
} else {
char* constraintstring = buildconstraintstring3(dapcomm->oc.dapconstraint);
nc_urisetconstraints(dapcomm->oc.url,constraintstring);
nullfree(constraintstring);
dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,NC_URICONSTRAINTS);
}
#ifdef DEBUG
fprintf(stderr,"ncdap3: final constraint: %s\n",dapcomm->oc.url->constraint);
#endif
/* Estimate the variable sizes */
estimatevarsizes3(dapcomm);
/* Build the meta data */
ncstat = buildncstructures3(dapcomm);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Do any necessary data prefetch */
if(FLAGSET(dapcomm->controls,NCF_PREFETCH)) {
ncstat = prefetchdata3(dapcomm);
if(ncstat != NC_NOERR) {
del_from_NCList((NC*)drno); /* undefine here */
{THROWCHK(ncstat); goto done;}
}
}
{
/* Mark as no longer writable and no longer indef;
requires breaking abstraction */
NC* nc;
ncstat = NC_check_id(drno->substrate, &nc);
/* Mark as no longer writeable */
fClr(nc->nciop->ioflags, NC_WRITE);
/* Mark as no longer indef;
(do NOT use nc_enddef until diskless is working)*/
fSet(nc->flags, NC_INDEF);
}
if(ncpp) *ncpp = (NC*)drno;
return ncstat;
done:
if(drno != NULL) NCD3_abort(drno->ext_ncid);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
NCerror
buildcachenode34(NCDAPCOMMON* nccomm,
DCEconstraint* constraint,
NClist* varlist,
NCcachenode** cachep,
NCFLAGS flags)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
OClink conn = nccomm->oc.conn;
OCddsnode ocroot = NULL;
CDFnode* dxdroot = NULL;
NCcachenode* cachenode = NULL;
char* ce = NULL;
int isprefetch = 0;
if((flags & NCF_PREFETCH) != 0)
isprefetch = 1;
if((flags & NCF_PREFETCH_ALL) == 0)
ce = buildconstraintstring3(constraint);
ocstat = dap_fetch(nccomm,conn,ce,OCDATADDS,&ocroot);
nullfree(ce);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
ncstat = buildcdftree34(nccomm,ocroot,OCDATA,&dxdroot);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* re-struct*/
if(!FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
ncstat = restruct3(dxdroot,nccomm->cdf.ddsroot,constraint->projections);
if(ncstat) {THROWCHK(ncstat); goto done;}
}
/* create the cache node */
cachenode = createnccachenode();
cachenode->isprefetch = isprefetch;
cachenode->vars = nclistclone(varlist);
cachenode->datadds = dxdroot;
/* Give the constraint over to the cachenode */
cachenode->constraint = constraint;
constraint = NULL;
cachenode->wholevariable = iscacheableconstraint(cachenode->constraint);
/* save the root content*/
cachenode->ocroot = ocroot;
ocstat = oc_data_getroot(conn,ocroot,&cachenode->content);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
/* capture the packet size */
ocstat = oc_raw_xdrsize(conn,ocroot,&cachenode->xdrsize);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
#ifdef DEBUG
fprintf(stderr,"buildcachenode: new cache node: %s\n",
dumpcachenode(cachenode));
#endif
/* Insert into the cache. If not caching, then
remove any previous cache node
*/
if(!isprefetch) {
NCcache* cache = nccomm->cdf.cache;
if(cache->nodes == NULL) cache->nodes = nclistnew();
/* remove cache nodes to get below the max cache size */
while(cache->cachesize + cachenode->xdrsize > cache->cachelimit
&& nclistlength(cache->nodes) > 0) {
NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0);
#ifdef DEBUG
fprintf(stderr,"buildcachenode: purge cache node: %s\n",
dumpcachenode(cachenode));
#endif
cache->cachesize -= node->xdrsize;
freenccachenode(nccomm,node);
}
/* Remove cache nodes to get below the max cache count */
/* If not caching, then cachecount should be 0 */
while(nclistlength(cache->nodes) > cache->cachecount) {
NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0);
#ifdef DEBUG
fprintf(stderr,"buildcachenode: count purge cache node: %s\n",
dumpcachenode(node));
#endif
cache->cachesize -= node->xdrsize;
freenccachenode(nccomm,node);
}
nclistpush(nccomm->cdf.cache->nodes,(void*)cachenode);
cache->cachesize += cachenode->xdrsize;
}
#ifdef DEBUG
fprintf(stderr,"buildcachenode: %s\n",dumpcachenode(cachenode));
#endif
done:
if(constraint != NULL) dcefree((DCEnode*)constraint);
if(cachep) *cachep = cachenode;
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
if(ncstat) {
freecdfroot34(dxdroot);
freenccachenode(nccomm,cachenode);
}
return THROW(ncstat);
}
