char* dumpcachenode(NCcachenode* node) { char* result = NULL; char tmp[8192]; int i; NCbytes* buf; if(node == NULL) return strdup("cachenode{null}"); buf = ncbytesnew(); snprintf(tmp,sizeof(tmp),"cachenode%s(%lx){size=%lu; constraint=%s; vars=", node->prefetch?"*":"", (unsigned long)node, (unsigned long)node->xdrsize, buildconstraintstring3(node->constraint)); ncbytescat(buf,tmp); if(nclistlength(node->vars)==0) ncbytescat(buf,"null"); else for(i=0;i<nclistlength(node->vars);i++) { CDFnode* var = (CDFnode*)nclistget(node->vars,i); if(i > 0) ncbytescat(buf,","); ncbytescat(buf,makesimplepathstring3(var)); } ncbytescat(buf,"}"); result = ncbytesdup(buf); ncbytesfree(buf); return result; }
NCerror fetchconstrainedmetadata3(NCDAPCOMMON* dapcomm) { NCerror ncstat = NC_NOERR; OCerror ocstat = OC_NOERR; OCddsnode ocroot; CDFnode* ddsroot; /* constrained */ char* ce = NULL; if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) ce = NULL; else ce = buildconstraintstring3(dapcomm->oc.dapconstraint); { ocstat = dap_fetch(dapcomm,dapcomm->oc.conn,ce,OCDDS,&ocroot); if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;} /* Construct our parallel dds tree; including attributes*/ ncstat = buildcdftree34(dapcomm,ocroot,OCDDS,&ddsroot); if(ncstat) goto fail; ocroot = NULL; /* avoid duplicate reclaim */ dapcomm->cdf.ddsroot = ddsroot; ddsroot = NULL; /* to avoid double reclamation */ if(!FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) { /* fix DAP server problem by adding back any inserting needed structure nodes */ ncstat = restruct3(dapcomm->cdf.ddsroot,dapcomm->cdf.fullddsroot,dapcomm->oc.dapconstraint->projections); if(ncstat) goto fail; } #ifdef DEBUG fprintf(stderr,"constrained:\n%s",dumptree(dapcomm->cdf.ddsroot)); #endif /* Combine DDS and DAS */ if(dapcomm->oc.ocdasroot != NULL) { ncstat = dapmerge3(dapcomm,dapcomm->cdf.ddsroot->ocnode, dapcomm->oc.ocdasroot); if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;} } /* map the constrained DDS to the unconstrained DDS */ ncstat = mapnodes3(dapcomm->cdf.ddsroot,dapcomm->cdf.fullddsroot); if(ncstat) goto fail; } fail: nullfree(ce); if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat); return ncstat; }
/* See ncd3dispatch.c for other version */ int NCD3_open(const char * path, int mode, int basepe, size_t *chunksizehintp, int useparallel, void* mpidata, NC_Dispatch* dispatch, NC** ncpp) { NCerror ncstat = NC_NOERR; OCerror ocstat = OC_NOERR; NC* drno = NULL; NCDAPCOMMON* dapcomm = NULL; const char* value; char* tmpname = NULL; if(!nc3dinitialized) nc3dinitialize(); if(path == NULL) return NC_EDAPURL; if(dispatch == NULL) PANIC("NC3D_open: no dispatch table"); /* Setup our NC and NCDAPCOMMON state*/ drno = (NC*)calloc(1,sizeof(NC)); if(drno == NULL) {ncstat = NC_ENOMEM; goto done;} /* compute an ncid */ ncstat = add_to_NCList(drno); if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} dapcomm = (NCDAPCOMMON*)calloc(1,sizeof(NCDAPCOMMON)); if(dapcomm == NULL) {ncstat = NC_ENOMEM; goto done;} drno->dispatch = dispatch; drno->dispatchdata = dapcomm; dapcomm->controller = (NC*)drno; dapcomm->cdf.separator = "."; dapcomm->cdf.smallsizelimit = DFALTSMALLLIMIT; dapcomm->cdf.cache = createnccache(); #ifdef HAVE_GETRLIMIT { struct rlimit rl; if(getrlimit(RLIMIT_NOFILE, &rl) >= 0) { dapcomm->cdf.cache->cachecount = (size_t)(rl.rlim_cur / 2); } } #endif #ifdef OCCOMPILEBYDEFAULT /* set the compile flag by default */ dapcomm->oc.rawurltext = (char*)emalloc(strlen(path)+strlen("[compile]")+1); strcpy(dapcomm->oc.rawurltext,"[compile]"); strcat(dapcomm->oc.rawurltext, path); #else dapcomm->oc.rawurltext = strdup(path); #endif nc_uriparse(dapcomm->oc.rawurltext,&dapcomm->oc.url); /* parse the client parameters */ nc_uridecodeparams(dapcomm->oc.url); if(!constrainable34(dapcomm->oc.url)) SETFLAG(dapcomm->controls,NCF_UNCONSTRAINABLE); /* Use libsrc code for storing metadata */ tmpname = nulldup(PSEUDOFILE); /* Now, use the file to create the netcdf file */ if(sizeof(size_t) == sizeof(unsigned int)) ncstat = nc_create(tmpname,NC_CLOBBER,&drno->substrate); else ncstat = nc_create(tmpname,NC_CLOBBER|NC_64BIT_OFFSET,&drno->substrate); if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} /* free the filename so it will automatically go away*/ unlink(tmpname); nullfree(tmpname); /* Avoid fill */ nc_set_fill(drno->substrate,NC_NOFILL,NULL); dapcomm->oc.dapconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT); dapcomm->oc.dapconstraint->projections = nclistnew(); dapcomm->oc.dapconstraint->selections = nclistnew(); /* Parse constraints to make sure they are syntactically correct */ ncstat = parsedapconstraints(dapcomm,dapcomm->oc.url->constraint,dapcomm->oc.dapconstraint); if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} /* Complain if we are unconstrainable but have constraints */ if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) { if(dapcomm->oc.url->constraint != NULL && strlen(dapcomm->oc.url->constraint) > 0) { nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s", dapcomm->oc.url->constraint); } } /* Construct a url for oc minus any parameters */ dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL, (NC_URIALL ^ NC_URICONSTRAINTS)); /* Pass to OC */ ocstat = oc_open(dapcomm->oc.urltext,&dapcomm->oc.conn); if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;} nullfree(dapcomm->oc.urltext); /* clean up */ dapcomm->oc.urltext = NULL; /* process control client parameters */ applyclientparamcontrols3(dapcomm); /* Turn on logging; only do this after oc_open*/ if((value = paramvalue34(dapcomm,"log")) != NULL) { ncloginit(); ncsetlogging(1); nclogopen(value); oc_loginit(); oc_setlogging(1); oc_logopen(value); } /* fetch and build the (almost) unconstrained DDS for use as template */ ncstat = fetchtemplatemetadata3(dapcomm); if(ncstat != NC_NOERR) goto done; /* fetch and build the constrained DDS */ ncstat = fetchconstrainedmetadata3(dapcomm); if(ncstat != NC_NOERR) goto done; #ifdef DEBUG2 fprintf(stderr,"constrained dds: %s\n",dumptree(dapcomm->cdf.ddsroot)); #endif /* The following actions are (mostly) WRT to the constrained tree */ /* Accumulate useful nodes sets */ ncstat = computecdfnodesets3(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* Fix grids */ ncstat = fixgrids3(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* Locate and mark usable sequences */ ncstat = sequencecheck3(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* suppress variables not in usable sequences */ ncstat = suppressunusablevars3(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* apply client parameters */ ncstat = applyclientparams34(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* Add (as needed) string dimensions*/ ncstat = addstringdims(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} if(nclistlength(dapcomm->cdf.seqnodes) > 0) { /* Build the sequence related dimensions */ ncstat = defseqdims(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} } /* Define the dimsetplus and dimsetall lists */ ncstat = definedimsets3(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* Re-compute the dimension names*/ ncstat = computecdfdimnames34(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* Deal with zero size dimensions */ ncstat = fixzerodims3(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} /* Attempt to use the DODS_EXTRA info to turn one of the dimensions into unlimited. Assume computecdfdimnames34 has already been called. */ ncstat = defrecorddim3(dapcomm); if(ncstat) {THROWCHK(ncstat); goto done;} if(dapcomm->cdf.recorddimname != NULL && nclistlength(dapcomm->cdf.seqnodes) > 0) { /*nclog(NCLOGWARN,"unlimited dimension specified, but sequences exist in DDS");*/ PANIC("unlimited dimension specified, but sequences exist in DDS"); } /* Re-compute the var names*/ ncstat = computecdfvarnames3(dapcomm,dapcomm->cdf.ddsroot,dapcomm->cdf.varnodes); if(ncstat) {THROWCHK(ncstat); goto done;} /* Transfer data from the unconstrained DDS data to the unconstrained DDS */ ncstat = dimimprint3(dapcomm); if(ncstat) goto done; /* Process the constraints to map to the constrained CDF tree */ /* (must follow fixgrids3 */ ncstat = mapconstraints3(dapcomm->oc.dapconstraint,dapcomm->cdf.ddsroot); if(ncstat != NC_NOERR) goto done; /* Canonicalize the constraint */ ncstat = fixprojections(dapcomm->oc.dapconstraint->projections); if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} /* Fill in segment information */ ncstat = qualifyconstraints3(dapcomm->oc.dapconstraint); if(ncstat != NC_NOERR) goto done; /* using the modified constraint, rebuild the constraint string */ if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) { /* ignore all constraints */ dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,0); } else { char* constraintstring = buildconstraintstring3(dapcomm->oc.dapconstraint); nc_urisetconstraints(dapcomm->oc.url,constraintstring); nullfree(constraintstring); dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,NC_URICONSTRAINTS); } #ifdef DEBUG fprintf(stderr,"ncdap3: final constraint: %s\n",dapcomm->oc.url->constraint); #endif /* Estimate the variable sizes */ estimatevarsizes3(dapcomm); /* Build the meta data */ ncstat = buildncstructures3(dapcomm); if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} /* Do any necessary data prefetch */ if(FLAGSET(dapcomm->controls,NCF_PREFETCH)) { ncstat = prefetchdata3(dapcomm); if(ncstat != NC_NOERR) { del_from_NCList((NC*)drno); /* undefine here */ {THROWCHK(ncstat); goto done;} } } { /* Mark as no longer writable and no longer indef; requires breaking abstraction */ NC* nc; ncstat = NC_check_id(drno->substrate, &nc); /* Mark as no longer writeable */ fClr(nc->nciop->ioflags, NC_WRITE); /* Mark as no longer indef; (do NOT use nc_enddef until diskless is working)*/ fSet(nc->flags, NC_INDEF); } if(ncpp) *ncpp = (NC*)drno; return ncstat; done: if(drno != NULL) NCD3_abort(drno->ext_ncid); if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat); return THROW(ncstat); }
NCerror buildcachenode34(NCDAPCOMMON* nccomm, DCEconstraint* constraint, NClist* varlist, NCcachenode** cachep, NCFLAGS flags) { NCerror ncstat = NC_NOERR; OCerror ocstat = OC_NOERR; OClink conn = nccomm->oc.conn; OCddsnode ocroot = NULL; CDFnode* dxdroot = NULL; NCcachenode* cachenode = NULL; char* ce = NULL; int isprefetch = 0; if((flags & NCF_PREFETCH) != 0) isprefetch = 1; if((flags & NCF_PREFETCH_ALL) == 0) ce = buildconstraintstring3(constraint); ocstat = dap_fetch(nccomm,conn,ce,OCDATADDS,&ocroot); nullfree(ce); if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} ncstat = buildcdftree34(nccomm,ocroot,OCDATA,&dxdroot); if(ncstat) {THROWCHK(ncstat); goto done;} /* re-struct*/ if(!FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) { ncstat = restruct3(dxdroot,nccomm->cdf.ddsroot,constraint->projections); if(ncstat) {THROWCHK(ncstat); goto done;} } /* create the cache node */ cachenode = createnccachenode(); cachenode->isprefetch = isprefetch; cachenode->vars = nclistclone(varlist); cachenode->datadds = dxdroot; /* Give the constraint over to the cachenode */ cachenode->constraint = constraint; constraint = NULL; cachenode->wholevariable = iscacheableconstraint(cachenode->constraint); /* save the root content*/ cachenode->ocroot = ocroot; ocstat = oc_data_getroot(conn,ocroot,&cachenode->content); if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} /* capture the packet size */ ocstat = oc_raw_xdrsize(conn,ocroot,&cachenode->xdrsize); if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} #ifdef DEBUG fprintf(stderr,"buildcachenode: new cache node: %s\n", dumpcachenode(cachenode)); #endif /* Insert into the cache. If not caching, then remove any previous cache node */ if(!isprefetch) { NCcache* cache = nccomm->cdf.cache; if(cache->nodes == NULL) cache->nodes = nclistnew(); /* remove cache nodes to get below the max cache size */ while(cache->cachesize + cachenode->xdrsize > cache->cachelimit && nclistlength(cache->nodes) > 0) { NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0); #ifdef DEBUG fprintf(stderr,"buildcachenode: purge cache node: %s\n", dumpcachenode(cachenode)); #endif cache->cachesize -= node->xdrsize; freenccachenode(nccomm,node); } /* Remove cache nodes to get below the max cache count */ /* If not caching, then cachecount should be 0 */ while(nclistlength(cache->nodes) > cache->cachecount) { NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0); #ifdef DEBUG fprintf(stderr,"buildcachenode: count purge cache node: %s\n", dumpcachenode(node)); #endif cache->cachesize -= node->xdrsize; freenccachenode(nccomm,node); } nclistpush(nccomm->cdf.cache->nodes,(void*)cachenode); cache->cachesize += cachenode->xdrsize; } #ifdef DEBUG fprintf(stderr,"buildcachenode: %s\n",dumpcachenode(cachenode)); #endif done: if(constraint != NULL) dcefree((DCEnode*)constraint); if(cachep) *cachep = cachenode; if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat); if(ncstat) { freecdfroot34(dxdroot); freenccachenode(nccomm,cachenode); } return THROW(ncstat); }