static OCerror
occorrelater(OCnode* dds, OCnode* dxd)
{
int i,j;
OCerror ocstat = OC_NOERR;
if(dds->octype != dxd->octype) {
THROWCHK((ocstat = OC_EINVAL)); goto fail;
}
if(dxd->name != NULL && dxd->name != NULL
&& strcmp(dxd->name,dds->name) != 0) {
THROWCHK((ocstat = OC_EINVAL)); goto fail;
} else if(dxd->name != dxd->name) { /* test NULL==NULL */
THROWCHK((ocstat = OC_EINVAL)); goto fail;
}
if(dxd->array.rank != dds->array.rank) {
THROWCHK((ocstat = OC_EINVAL)); goto fail;
}
dds->datadds = dxd;
switch (dds->octype) {
case OC_Dataset:
case OC_Structure:
case OC_Grid:
case OC_Sequence:
/* Remember: there may be fewer datadds fields than dds fields */
for(i=0;i<oclistlength(dxd->subnodes);i++) {
OCnode* dxd1 = (OCnode*)oclistget(dxd->subnodes,i);
for(j=0;j<oclistlength(dds->subnodes);j++) {
OCnode* dds1 = (OCnode*)oclistget(dds->subnodes,j);
if(strcmp(dxd1->name,dds1->name) == 0) {
ocstat = occorrelater(dds1,dxd1);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;}
break;
}
}
}
break;
case OC_Dimension:
case OC_Primitive:
break;
default: OCPANIC1("unexpected node type: %d",dds->octype);
}
/* Correlate the dimensions */
if(dds->array.rank > 0) {
for(i=0;i<oclistlength(dxd->subnodes);i++) {
OCnode* ddsdim = (OCnode*)oclistget(dds->array.dimensions,i);
OCnode* dxddim = (OCnode*)oclistget(dxd->array.dimensions,i);
ocstat = occorrelater(ddsdim,dxddim);
if(!ocstat) goto fail;
}
}
fail:
return THROW(ocstat);
}
NCerror
fetchtemplatemetadata3(NCDAPCOMMON* dapcomm)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
OCddsnode ocroot = NULL;
CDFnode* ddsroot = NULL;
char* ce = NULL;
/* Temporary hack: we need to get the selection string
from the url
*/
/* Get (almost) unconstrained DDS; In order to handle functions
correctly, those selections must always be included
*/
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE))
ce = NULL;
else
ce = nulldup(dapcomm->oc.url->selection);
/* Get selection constrained DDS */
ocstat = dap_fetch(dapcomm,dapcomm->oc.conn,ce,OCDDS,&ocroot);
if(ocstat != OC_NOERR) {
/* Special Hack. If the protocol is file, then see if
we can get the dds from the .dods file
*/
if(strcmp(dapcomm->oc.url->protocol,"file") != 0) {
THROWCHK(ocstat); goto done;
}
/* Fetch the data dds */
ocstat = dap_fetch(dapcomm,dapcomm->oc.conn,ce,OCDATADDS,&ocroot);
if(ocstat != OC_NOERR) {
THROWCHK(ocstat); goto done;
}
/* Note what we did */
nclog(NCLOGWARN,"Cannot locate .dds file, using .dods file");
}
/* Get selection constrained DAS */
ocstat = dap_fetch(dapcomm,dapcomm->oc.conn,ce,OCDAS,&dapcomm->oc.ocdasroot);
if(ocstat != OC_NOERR) {
/* Ignore but complain */
nclog(NCLOGWARN,"Could not read DAS; ignored");
dapcomm->oc.ocdasroot = NULL;
ocstat = OC_NOERR;
}
/* Construct the netcdf cdf tree corresponding to the dds tree*/
ncstat = buildcdftree34(dapcomm,ocroot,OCDDS,&ddsroot);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
dapcomm->cdf.fullddsroot = ddsroot;
done:
nullfree(ce);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return ncstat;
}
NCerror
fetchconstrainedmetadata3(NCDAPCOMMON* dapcomm)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
OCddsnode ocroot;
CDFnode* ddsroot; /* constrained */
char* ce = NULL;
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE))
ce = NULL;
else
ce = buildconstraintstring3(dapcomm->oc.dapconstraint);
{
ocstat = dap_fetch(dapcomm,dapcomm->oc.conn,ce,OCDDS,&ocroot);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;}
/* Construct our parallel dds tree; including attributes*/
ncstat = buildcdftree34(dapcomm,ocroot,OCDDS,&ddsroot);
if(ncstat) goto fail;
ocroot = NULL; /* avoid duplicate reclaim */
dapcomm->cdf.ddsroot = ddsroot;
ddsroot = NULL; /* to avoid double reclamation */
if(!FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
/* fix DAP server problem by adding back any inserting needed structure nodes */
ncstat = restruct3(dapcomm->cdf.ddsroot,dapcomm->cdf.fullddsroot,dapcomm->oc.dapconstraint->projections);
if(ncstat) goto fail;
}
#ifdef DEBUG
fprintf(stderr,"constrained:\n%s",dumptree(dapcomm->cdf.ddsroot));
#endif
/* Combine DDS and DAS */
if(dapcomm->oc.ocdasroot != NULL) {
ncstat = dapmerge3(dapcomm,dapcomm->cdf.ddsroot->ocnode,
dapcomm->oc.ocdasroot);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
}
/* map the constrained DDS to the unconstrained DDS */
ncstat = mapnodes3(dapcomm->cdf.ddsroot,dapcomm->cdf.fullddsroot);
if(ncstat) goto fail;
}
fail:
nullfree(ce);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return ncstat;
}
static NCerror
moveto(NCDAPCOMMON* nccomm, Getvara* xgetvar, CDFnode* xrootnode, void* memory)
{
OCerror ocstat = OC_NOERR;
NCerror ncstat = NC_NOERR;
OClink conn = nccomm->oc.conn;
OCdatanode xrootcontent;
OCddsnode ocroot;
NClist* path = nclistnew();
struct NCMEMORY memstate;
memstate.next = (memstate.memory = memory);
/* Get the root content*/
ocroot = xrootnode->tree->ocroot;
ocstat = oc_data_getroot(conn,ocroot,&xrootcontent);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
/* Remember: xgetvar->target is in DATADDS tree */
collectnodepath3(xgetvar->target,path,WITHDATASET);
ncstat = movetor(nccomm,xrootcontent,
path,0,xgetvar,0,&memstate,
xgetvar->varaprojection->var->segments);
done:
nclistfree(path);
oc_data_free(conn,xrootcontent);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
OCerror
ocopen(OCstate** statep, const char* url)
{
int stat = OC_NOERR;
OCstate * state = NULL;
DAPURL tmpurl;
CURL* curl = NULL; /* curl handle*/
memset((void*)&tmpurl,0,sizeof(tmpurl));
if(!dapurlparse(url,&tmpurl)) {THROWCHK(stat=OC_EBADURL); goto fail;}
stat = occurlopen(&curl);
if(stat != OC_NOERR) {THROWCHK(stat); goto fail;}
state = (OCstate*)ocmalloc(sizeof(OCstate)); /* ocmalloc zeros memory*/
if(state == NULL) {THROWCHK(stat=OC_ENOMEM); goto fail;}
/* Setup DAP state*/
state->magic = OCMAGIC;
state->curl = curl;
state->trees = oclistnew();
state->contentlist = NULL;
state->url = tmpurl;
state->clientparams = ocparamdecode(state->url.params);
if(state->clientparams == NULL) {
oc_log(LOGWARN,"Could not parse client parameters");
}
state->packet = ocbytesnew();
ocbytessetalloc(state->packet,DFALTPACKETSIZE); /*initial reasonable size*/
/* set curl properties for this link */
ocsetcurlproperties(state);
if(statep) *statep = state;
return THROW(stat);
fail:
dapurlclear(&tmpurl);
if(state != NULL) ocfree(state);
if(curl != NULL) occurlclose(curl);
return THROW(stat);
}
static NCerror
movetofield(NCDAPCOMMON* nccomm,
OCdatanode currentcontent,
NClist* path,
int depth, /* depth is position in segment list*/
Getvara* xgetvar,
size_t dimindex, /* dimindex is position in xgetvar->slices*/
struct NCMEMORY* memory,
NClist* segments)
{
OCerror ocstat = OC_NOERR;
NCerror ncstat = NC_NOERR;
size_t fieldindex,gridindex;
OClink conn = nccomm->oc.conn;
CDFnode* xnode = (CDFnode*)nclistget(path,depth);
OCdatanode reccontent = NULL;
OCdatanode dimcontent = NULL;
OCdatanode fieldcontent = NULL;
CDFnode* xnext;
int newdepth;
/* currentcontent points to the grid/dataset/structure/record instance */
xnext = (CDFnode*)nclistget(path,depth+1);
ASSERT((xnext != NULL));
fieldindex = findfield(xnode,xnext);
/* If the next node is a nc_virtual node, then
we need to effectively
ignore it and use the appropriate subnode.
If the next node is a re-struct'd node, then
use it as is.
*/
if(xnext->nc_virtual) {
CDFnode* xgrid = xnext;
xnext = (CDFnode*)nclistget(path,depth+2); /* real node */
gridindex = fieldindex;
fieldindex = findfield(xgrid,xnext);
fieldindex += gridindex;
newdepth = depth+2;
} else {
newdepth = depth+1;
}
/* Move to appropriate field */
ocstat = oc_data_ithfield(conn,currentcontent,fieldindex,&fieldcontent);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
ncstat = movetor(nccomm,fieldcontent,
path,newdepth,xgetvar,dimindex,memory,
segments);
done:
oc_data_free(conn,dimcontent);
oc_data_free(conn,fieldcontent);
oc_data_free(conn,reccontent);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
NCerror
mapnodes(CDFnode* root, CDFnode* fullroot)
{
NCerror ncstat = NC_NOERR;
ASSERT(root != NULL && fullroot != NULL);
if(!simplenodematch(root,fullroot))
{THROWCHK(ncstat=NC_EINVAL); goto done;}
/* clear out old associations*/
unmap(root);
ncstat = mapnodesr(root,fullroot,0);
done:
return ncstat;
}
OCerror
oc_raw_xdrsize(OCconnection conn, OCobject root0, size_t* sizep)
{
OCstate* state;
OCnode* root;
OCerror ocerr = OC_NOERR;
OCVERIFY(OCstate*,state,conn);
OCDEREF(OCstate*,state,conn);
OCVERIFY(OCnode*,root,root0);
OCDEREF(OCnode*,root,root0);
if(sizep == NULL) goto done;
if(root->tree == NULL || root->tree->dxdclass != OCDATADDS)
{THROWCHK((ocerr=OC_EINVAL)); goto done;}
if(sizep) *sizep = root->tree->data.datasize;
done:
return ocerr;
}
OCerror
oc_data_root(OCconnection conn, OCobject root0, OCdata content0)
{
OCstate* state;
OCnode* root;
OCcontent* content;
OCerror ocerr = OC_NOERR;
OCVERIFY(OCstate*,state,conn);
OCDEREF(OCstate*,state,conn);
OCVERIFY(OCnode*,root,root0);
OCDEREF(OCnode*,root,root0);
OCVERIFY(OCcontent*,content,content0);
OCDEREF(OCcontent*,content,content0);
if(root->tree == NULL) {THROWCHK((ocerr=OC_EINVAL)); goto fail;}
ocerr = ocrootcontent(state,root,content);
fail:
return ocerr;
}
/* Compute the set of prefetched data.
Notes:
1. Even if caching is off, we will
still prefetch the small variables.
2. All prefetches are whole variable fetches.
3. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
int isnc4 = FLAGSET(nccomm->controls,NCF_NC4);
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else { /* can do constraints */
/* pull in those variables of sufficiently small size */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
size_t nelems = 1;
if(!isnc4) {
/* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */
if(var->nctype == NC_Sequence || dapinsequence(var)) continue;
}
/* Compute the # of elements in the variable */
for(j=0;j<nclistlength(var->array.dimset0);j++) {
CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j);
nelems *= dim->dim.declsize;
}
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems);
}
if(nelems <= nccomm->cdf.smallsizelimit) {
nclistpush(vars,(ncelem)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(ncelem)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
NCerror
dapconvert3(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t count)
{
NCerror ncstat = NC_NOERR;
size_t i;
char* memory = memory0;
char* value = value0;
/* In order to deal with the DAP upgrade problem,
try to preserve the bit patterns
*/
/* Provide space and pointer casts for intermediate results */
signed char ncbyte;
signed char* ncbytep;
char ncchar;
char* nccharp;
short ncshort;
short* ncshortp;
int ncint;
int* ncintp;
float ncfloat;
float* ncfloatp;
double ncdouble;
double* ncdoublep;
unsigned char ncubyte;
unsigned char* ncubytep;
unsigned short ncushort;
unsigned short* ncushortp;
unsigned int ncuint;
unsigned int* ncuintp;
long long ncint64;
long long* ncint64p;
unsigned long long ncuint64;
unsigned long long* ncuint64p;
#define CASE(nc1,nc2) (nc1*256+nc2)
#define CUT8(e) ((unsigned char)((e) & 0xff))
#define CUT16(e) ((unsigned short)((e) & 0xffff))
#define CUT32(e) ((unsigned int)((e) & 0xffffffff))
#define ARM(vs,ncs,ts,vd,ncd,td) \
case CASE(ncs,ncd):\
vs##p = (ts *)value;\
vs = *vs##p;\
vd##p = (td *)memory;\
*vd##p = (td)vs;\
break;
for(i=0;i<count;i++) {
switch (CASE(srctype,dsttype)) {
ARM(ncchar,NC_CHAR,char,ncchar,NC_CHAR,char)
ARM(ncchar,NC_CHAR,char,ncbyte,NC_BYTE,signed char)
ARM(ncchar,NC_CHAR,char,ncubyte,NC_UBYTE,unsigned char)
ARM(ncchar,NC_CHAR,char,ncshort,NC_SHORT,short)
ARM(ncchar,NC_CHAR,char,ncushort,NC_USHORT,unsigned short)
ARM(ncchar,NC_CHAR,char,ncint,NC_INT,int)
ARM(ncchar,NC_CHAR,char,ncuint,NC_UINT,unsigned int)
ARM(ncchar,NC_CHAR,char,ncint64,NC_INT64,long long)
ARM(ncchar,NC_CHAR,char,ncuint64,NC_UINT64,unsigned long long)
ARM(ncchar,NC_CHAR,char,ncfloat,NC_FLOAT,float)
ARM(ncchar,NC_CHAR,char,ncdouble,NC_DOUBLE,double)
ARM(ncbyte,NC_BYTE,signed char,ncchar,NC_CHAR,char)
ARM(ncbyte,NC_BYTE,signed char,ncbyte,NC_BYTE,signed char)
ARM(ncbyte,NC_BYTE,signed char,ncubyte,NC_UBYTE,unsigned char)
ARM(ncbyte,NC_BYTE,signed char,ncshort,NC_SHORT,short)
ARM(ncbyte,NC_BYTE,signed char,ncushort,NC_USHORT,unsigned short)
ARM(ncbyte,NC_BYTE,signed char,ncint,NC_INT,int)
ARM(ncbyte,NC_BYTE,signed char,ncuint,NC_UINT,unsigned int)
ARM(ncbyte,NC_BYTE,signed char,ncint64,NC_INT64,long long)
ARM(ncbyte,NC_BYTE,signed char,ncuint64,NC_UINT64,unsigned long long)
ARM(ncbyte,NC_BYTE,signed char,ncfloat,NC_FLOAT,float)
ARM(ncbyte,NC_BYTE,signed char,ncdouble,NC_DOUBLE,double)
ARM(ncubyte,NC_UBYTE,unsigned char,ncchar,NC_CHAR,char)
ARM(ncubyte,NC_UBYTE,unsigned char,ncbyte,NC_BYTE,signed char)
ARM(ncubyte,NC_UBYTE,unsigned char,ncubyte,NC_UBYTE,unsigned char)
ARM(ncubyte,NC_UBYTE,unsigned char,ncshort,NC_SHORT,short)
ARM(ncubyte,NC_UBYTE,unsigned char,ncushort,NC_USHORT,unsigned short)
ARM(ncubyte,NC_UBYTE,unsigned char,ncint,NC_INT,int)
ARM(ncubyte,NC_UBYTE,unsigned char,ncuint,NC_UINT,unsigned int)
ARM(ncubyte,NC_UBYTE,unsigned char,ncint64,NC_INT64,long long)
ARM(ncubyte,NC_UBYTE,unsigned char,ncuint64,NC_UINT64,unsigned long long)
ARM(ncubyte,NC_UBYTE,unsigned char,ncfloat,NC_FLOAT,float)
ARM(ncubyte,NC_UBYTE,unsigned char,ncdouble,NC_DOUBLE,double)
ARM(ncshort,NC_SHORT,short,ncchar,NC_CHAR,char)
ARM(ncshort,NC_SHORT,short,ncbyte,NC_BYTE,signed char)
ARM(ncshort,NC_SHORT,short,ncubyte,NC_UBYTE,unsigned char)
ARM(ncshort,NC_SHORT,short,ncshort,NC_SHORT,short)
ARM(ncshort,NC_SHORT,short,ncushort,NC_USHORT,unsigned short)
ARM(ncshort,NC_SHORT,short,ncint,NC_INT,int)
ARM(ncshort,NC_SHORT,short,ncuint,NC_UINT,unsigned int)
ARM(ncshort,NC_SHORT,short,ncint64,NC_INT64,long long)
ARM(ncshort,NC_SHORT,short,ncuint64,NC_UINT64,unsigned long long)
ARM(ncshort,NC_SHORT,short,ncfloat,NC_FLOAT,float)
ARM(ncshort,NC_SHORT,short,ncdouble,NC_DOUBLE,double)
ARM(ncushort,NC_USHORT,unsigned short,ncchar,NC_CHAR,char)
ARM(ncushort,NC_USHORT,unsigned short,ncbyte,NC_BYTE,signed char)
ARM(ncushort,NC_USHORT,unsigned short,ncubyte,NC_UBYTE,unsigned char)
ARM(ncushort,NC_USHORT,unsigned short,ncshort,NC_SHORT,short)
ARM(ncushort,NC_USHORT,unsigned short,ncushort,NC_USHORT,unsigned short)
ARM(ncushort,NC_USHORT,unsigned short,ncint,NC_INT,int)
ARM(ncushort,NC_USHORT,unsigned short,ncuint,NC_UINT,unsigned int)
ARM(ncushort,NC_USHORT,unsigned short,ncint64,NC_INT64,long long)
ARM(ncushort,NC_USHORT,unsigned short,ncuint64,NC_UINT64,unsigned long long)
ARM(ncushort,NC_USHORT,unsigned short,ncfloat,NC_FLOAT,float)
ARM(ncushort,NC_USHORT,unsigned short,ncdouble,NC_DOUBLE,double)
ARM(ncint,NC_INT,int,ncchar,NC_CHAR,char)
ARM(ncint,NC_INT,int,ncbyte,NC_BYTE,signed char)
ARM(ncint,NC_INT,int,ncubyte,NC_UBYTE,unsigned char)
ARM(ncint,NC_INT,int,ncshort,NC_SHORT,short)
ARM(ncint,NC_INT,int,ncushort,NC_USHORT,unsigned short)
ARM(ncint,NC_INT,int,ncint,NC_INT,int)
ARM(ncint,NC_INT,int,ncuint,NC_UINT,unsigned int)
ARM(ncint,NC_INT,int,ncint64,NC_INT64,long long)
ARM(ncint,NC_INT,int,ncuint64,NC_UINT64,unsigned long long)
ARM(ncint,NC_INT,int,ncfloat,NC_FLOAT,float)
ARM(ncint,NC_INT,int,ncdouble,NC_DOUBLE,double)
ARM(ncuint,NC_UINT,unsigned int,ncchar,NC_CHAR,char)
ARM(ncuint,NC_UINT,unsigned int,ncbyte,NC_BYTE,signed char)
ARM(ncuint,NC_UINT,unsigned int,ncubyte,NC_UBYTE,unsigned char)
ARM(ncuint,NC_UINT,unsigned int,ncshort,NC_SHORT,short)
ARM(ncuint,NC_UINT,unsigned int,ncushort,NC_USHORT,unsigned short)
ARM(ncuint,NC_UINT,unsigned int,ncint,NC_INT,int)
ARM(ncuint,NC_UINT,unsigned int,ncuint,NC_UINT,unsigned int)
ARM(ncuint,NC_UINT,unsigned int,ncint64,NC_INT64,long long)
ARM(ncuint,NC_UINT,unsigned int,ncuint64,NC_UINT64,unsigned long long)
ARM(ncuint,NC_UINT,unsigned int,ncfloat,NC_FLOAT,float)
ARM(ncuint,NC_UINT,unsigned int,ncdouble,NC_DOUBLE,double)
ARM(ncint64,NC_INT64,long long,ncchar,NC_CHAR,char)
ARM(ncint64,NC_INT64,long long,ncbyte,NC_BYTE,signed char)
ARM(ncint64,NC_INT64,long long,ncubyte,NC_UBYTE,unsigned char)
ARM(ncint64,NC_INT64,long long,ncshort,NC_SHORT,short)
ARM(ncint64,NC_INT64,long long,ncushort,NC_USHORT,unsigned short)
ARM(ncint64,NC_INT64,long long,ncint,NC_INT,int)
ARM(ncint64,NC_INT64,long long,ncuint,NC_UINT,unsigned int)
ARM(ncint64,NC_INT64,long long,ncint64,NC_INT64,long long)
ARM(ncint64,NC_INT64,long long,ncuint64,NC_UINT64,unsigned long long)
ARM(ncint64,NC_INT64,long long,ncfloat,NC_FLOAT,float)
ARM(ncint64,NC_INT64,long long,ncdouble,NC_DOUBLE,double)
ARM(ncuint64,NC_UINT64,unsigned long long,ncchar,NC_CHAR,char)
ARM(ncuint64,NC_UINT64,unsigned long long,ncbyte,NC_BYTE,signed char)
ARM(ncuint64,NC_UINT64,unsigned long long,ncubyte,NC_UBYTE,unsigned char)
ARM(ncuint64,NC_UINT64,unsigned long long,ncshort,NC_SHORT,short)
ARM(ncuint64,NC_UINT64,unsigned long long,ncushort,NC_USHORT,unsigned short)
ARM(ncuint64,NC_UINT64,unsigned long long,ncint,NC_INT,int)
ARM(ncuint64,NC_UINT64,unsigned long long,ncuint,NC_UINT,unsigned int)
ARM(ncuint64,NC_UINT64,unsigned long long,ncint64,NC_INT64,long long)
ARM(ncuint64,NC_UINT64,unsigned long long,ncuint64,NC_UINT64,unsigned long long)
ARM(ncuint64,NC_UINT64,unsigned long long,ncfloat,NC_FLOAT,float)
ARM(ncuint64,NC_UINT64,unsigned long long,ncdouble,NC_DOUBLE,double)
ARM(ncfloat,NC_FLOAT,float,ncchar,NC_CHAR,char)
ARM(ncfloat,NC_FLOAT,float,ncbyte,NC_BYTE,signed char)
ARM(ncfloat,NC_FLOAT,float,ncubyte,NC_UBYTE,unsigned char)
ARM(ncfloat,NC_FLOAT,float,ncshort,NC_SHORT,short)
ARM(ncfloat,NC_FLOAT,float,ncushort,NC_USHORT,unsigned short)
ARM(ncfloat,NC_FLOAT,float,ncint,NC_INT,int)
ARM(ncfloat,NC_FLOAT,float,ncuint,NC_UINT,unsigned int)
ARM(ncfloat,NC_FLOAT,float,ncint64,NC_INT64,long long)
ARM(ncfloat,NC_FLOAT,float,ncuint64,NC_UINT64,unsigned long long)
ARM(ncfloat,NC_FLOAT,float,ncfloat,NC_FLOAT,float)
ARM(ncfloat,NC_FLOAT,float,ncdouble,NC_DOUBLE,double)
ARM(ncdouble,NC_DOUBLE,double,ncchar,NC_CHAR,char)
ARM(ncdouble,NC_DOUBLE,double,ncbyte,NC_BYTE,signed char)
ARM(ncdouble,NC_DOUBLE,double,ncubyte,NC_UBYTE,unsigned char)
ARM(ncdouble,NC_DOUBLE,double,ncshort,NC_SHORT,short)
ARM(ncdouble,NC_DOUBLE,double,ncushort,NC_USHORT,unsigned short)
ARM(ncdouble,NC_DOUBLE,double,ncint,NC_INT,int)
ARM(ncdouble,NC_DOUBLE,double,ncuint,NC_UINT,unsigned int)
ARM(ncdouble,NC_DOUBLE,double,ncint64,NC_INT64,long long)
ARM(ncdouble,NC_DOUBLE,double,ncuint64,NC_UINT64,unsigned long long)
ARM(ncdouble,NC_DOUBLE,double,ncfloat,NC_FLOAT,float)
ARM(ncdouble,NC_DOUBLE,double,ncdouble,NC_DOUBLE,double)
default: ncstat = NC_EINVAL; THROWCHK(ncstat); goto fail;
}
value += nctypesizeof(srctype);
memory += nctypesizeof(dsttype);
}
fail:
return THROW(ncstat);
}
/* We are at a primitive variable or scalar that has no string dimensions.
Extract the data. (This is way too complicated)
*/
static int
extract(
NCDAPCOMMON* nccomm,
Getvara* xgetvar,
CDFnode* xnode,
DCEsegment* segment,
size_t dimindex,/*notused*/
OClink conn,
OCdatanode currentcontent,
struct NCMEMORY* memory
)
{
OCerror ocstat = OC_NOERR;
NCerror ncstat = NC_NOERR;
size_t count,rank0;
Dapodometer* odom = NULL;
size_t externtypesize;
size_t interntypesize;
int requireconversion;
char value[16];
ASSERT((segment != NULL));
requireconversion = conversionrequired(xgetvar->dsttype,xnode->etype);
ASSERT(xgetvar->cache != NULL);
externtypesize = nctypesizeof(xgetvar->dsttype);
interntypesize = nctypesizeof(xnode->etype);
rank0 = nclistlength(xnode->array.dimset0);
#ifdef DEBUG2
fprintf(stderr,"moveto: primitive: segment=%s",
dcetostring((DCEnode*)segment));
fprintf(stderr," iswholevariable=%d",xgetvar->cache->wholevariable);
fprintf(stderr,"\n");
#endif
if(rank0 == 0) {/* scalar */
char* mem = (requireconversion?value:memory->next);
ASSERT(externtypesize <= sizeof(value));
/* Read the whole scalar directly into memory */
ocstat = oc_data_readscalar(conn,currentcontent,externtypesize,mem);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
if(requireconversion) {
/* convert the value to external type */
ncstat = dapconvert3(xnode->etype,xgetvar->dsttype,memory->next,value,1);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
}
memory->next += (externtypesize);
} else if(xgetvar->cache->wholevariable) {/* && rank0 > 0 */
/* There are multiple cases, assuming no conversion required.
1) client is asking for whole variable
=> start=0, count=totalsize, stride=1
=> read whole thing at one shot
2) client is asking for non-strided subset
and edges are maximal
=> start=x, count=y, stride=1
=> read whole subset at one shot
3) client is asking for strided subset or edges are not maximal
=> start=x, count=y, stride=s
=> we have to use odometer on leading prefix.
If conversion required, then read one-by-one
*/
int safeindex = dcesafeindex(segment,0,rank0);
assert(safeindex >= 0 && safeindex <= rank0);
if(!requireconversion && safeindex == 0) { /* can read whole thing */
size_t internlen;
count = dcesegmentsize(segment,0,rank0); /* how many to read */
internlen = interntypesize*count;
/* Read the whole variable directly into memory.*/
ocstat = oc_data_readn(conn,currentcontent,dap_zero,count,internlen,memory->next);
/* bump memory pointer */
memory->next += internlen;
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
} else if(!requireconversion && safeindex > 0 && safeindex < rank0) {
size_t internlen;
/* We need to build an odometer for the unsafe prefix of the slices */
odom = dapodom_fromsegment(segment,0,safeindex);
count = dcesegmentsize(segment,safeindex,rank0); /* read in count chunks */
internlen = interntypesize*count;
while(dapodom_more(odom)) {
ocstat = oc_data_readn(conn,currentcontent,odom->index,count,internlen,memory->next);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
memory->next += internlen;
dapodom_next(odom);
}
dapodom_free(odom);
} else {
/* Cover following cases, all of which require reading
values one-by-one:
1. requireconversion
2. !requireconversion but safeindex == rank0 =>no safe indices
Note that in case 2, we will do a no-op conversion.
*/
odom = dapodom_fromsegment(segment,0,rank0);
while(dapodom_more(odom)) {
char value[16]; /* Big enough to hold any numeric value */
ocstat = oc_data_readn(conn,currentcontent,odom->index,1,interntypesize,value);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
ncstat = dapconvert3(xnode->etype,xgetvar->dsttype,memory->next,value,1);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
memory->next += (externtypesize);
dapodom_next(odom);
}
dapodom_free(odom);
}
} else { /* !xgetvar->cache->wholevariable && rank0 > 0 */
/* This is the case where the constraint was applied by the server,
so we just read it in, possibly with conversion
*/
if(requireconversion) {
/* read one-by-one */
odom = dapodom_fromsegment(segment,0,rank0);
while(dapodom_more(odom)) {
char value[16]; /* Big enough to hold any numeric value */
ocstat = oc_data_readn(conn,currentcontent,odom->index,1,interntypesize,value);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
ncstat = dapconvert3(xnode->etype,xgetvar->dsttype,memory->next,value,1);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
memory->next += (externtypesize);
dapodom_next(odom);
}
dapodom_free(odom);
} else {/* Read straight to memory */
size_t internlen;
count = dcesegmentsize(segment,0,rank0); /* how many to read */
internlen = interntypesize*count;
ocstat = oc_data_readn(conn,currentcontent,dap_zero,count,internlen,memory->next);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
}
}
done:
return THROW(ncstat);
}
NCerror
nc3d_getvarx(int ncid, int varid,
const size_t *startp,
const size_t *countp,
const ptrdiff_t* stridep,
void *data,
nc_type dsttype0)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
int i;
NC* drno;
NC* substrate;
NCDAPCOMMON* dapcomm;
CDFnode* cdfvar = NULL; /* cdf node mapping to var*/
NClist* varnodes;
nc_type dsttype;
Getvara* varainfo = NULL;
CDFnode* xtarget = NULL; /* target in DATADDS */
CDFnode* target = NULL; /* target in constrained DDS */
DCEprojection* varaprojection = NULL;
NCcachenode* cachenode = NULL;
size_t localcount[NC_MAX_VAR_DIMS];
NClist* ncdimsall;
size_t ncrank;
NClist* vars = NULL;
DCEconstraint* fetchconstraint = NULL;
DCEprojection* fetchprojection = NULL;
DCEprojection* walkprojection = NULL;
int state;
#define FETCHWHOLE 1 /* fetch whole data set */
#define FETCHVAR 2 /* fetch whole variable */
#define FETCHPART 4 /* fetch constrained variable */
#define CACHED 8 /* whole variable is already in the cache */
ncstat = NC_check_id(ncid, (NC**)&drno);
if(ncstat != NC_NOERR) goto fail;
dapcomm = (NCDAPCOMMON*)drno->dispatchdata;
ncstat = NC_check_id(drno->substrate, (NC**)&substrate);
if(ncstat != NC_NOERR) goto fail;
/* Locate var node via varid */
varnodes = dapcomm->cdf.ddsroot->tree->varnodes;
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(varnodes,i);
if(node->array.basevar == NULL
&& node->nctype == NC_Atomic
&& node->ncid == varid) {
cdfvar = node;
break;
}
}
ASSERT((cdfvar != NULL));
/* If the variable is prefetchable, then now
is the time to do a lazy prefetch */
if(FLAGSET(dapcomm->controls,NCF_PREFETCH)
&& !FLAGSET(dapcomm->controls,NCF_PREFETCH_EAGER)) {
if(dapcomm->cdf.cache != NULL && dapcomm->cdf.cache->prefetch == NULL) {
ncstat = prefetchdata3(dapcomm);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
}
}
/* Get the dimension info */
ncdimsall = cdfvar->array.dimsetall;
ncrank = nclistlength(ncdimsall);
#ifdef DEBUG
{
int i;
fprintf(stderr,"getvarx: %s",cdfvar->ncfullname);
for(i=0;i<ncrank;i++)
fprintf(stderr,"(%ld:%ld:%ld)",
(long)startp[i],
(long)countp[i],
(long)stridep[i]
);
fprintf(stderr,"\n");
}
#endif
/* Fill in missing arguments */
if(startp == NULL)
startp = nc_sizevector0;
if(countp == NULL) {
/* Accumulate the dimension sizes */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdimsall,i);
localcount[i] = dim->dim.declsize;
}
countp = localcount;
}
if(stridep == NULL)
stridep = nc_ptrdiffvector1;
/* Validate the dimension sizes */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdimsall,i);
if(startp[i] > dim->dim.declsize
|| startp[i]+countp[i] > dim->dim.declsize) {
ncstat = NC_EINVALCOORDS;
goto fail;
}
}
#ifdef DEBUG
{
NClist* dims = cdfvar->array.dimsetall;
fprintf(stderr,"getvarx: %s",cdfvar->ncfullname);
if(nclistlength(dims) > 0) {int i;
for(i=0;i<nclistlength(dims);i++)
fprintf(stderr,"(%lu:%lu:%lu)",(unsigned long)startp[i],(unsigned long)countp[i],(unsigned long)stridep[i]);
fprintf(stderr," -> ");
for(i=0;i<nclistlength(dims);i++)
if(stridep[i]==1)
fprintf(stderr,"[%lu:%lu]",(unsigned long)startp[i],(unsigned long)((startp[i]+countp[i])-1));
else {
unsigned long iend = (stridep[i] * countp[i]);
iend = (iend + startp[i]);
iend = (iend - 1);
fprintf(stderr,"[%lu:%lu:%lu]",
(unsigned long)startp[i],(unsigned long)stridep[i],iend);
}
}
fprintf(stderr,"\n");
}
#endif
dsttype = (dsttype0);
/* Default to using the inquiry type for this var*/
if(dsttype == NC_NAT) dsttype = cdfvar->externaltype;
/* Validate any implied type conversion*/
if(cdfvar->etype != dsttype && dsttype == NC_CHAR) {
/* The only disallowed conversion is to/from char and non-byte
numeric types*/
switch (cdfvar->etype) {
case NC_STRING: case NC_URL:
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
break;
default:
return THROW(NC_ECHAR);
}
}
ncstat = makegetvar34(dapcomm,cdfvar,data,dsttype,&varainfo);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Compile the start/stop/stride info into a projection */
ncstat = buildvaraprojection3(varainfo->target,
startp,countp,stridep,
&varaprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
fetchprojection = NULL;
walkprojection = NULL;
/* Create walkprojection as the merge of the url projections
and the vara projection; may change in FETCHPART case below*/
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&walkprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
#ifdef DEBUG
fprintf(stderr,"getvarx: walkprojection: |%s|\n",dumpprojection(walkprojection));
#endif
/* define the var list of interest */
vars = nclistnew();
nclistpush(vars,(void*)varainfo->target);
state = 0;
if(iscached(dapcomm,cdfvar,&cachenode)) { /* ignores non-whole variable cache entries */
state = CACHED;
ASSERT((cachenode != NULL));
#ifdef DEBUG
fprintf(stderr,"var is in cache\n");
#endif
/* If it is cached, then it is a whole variable but may still
need to apply constraints during the walk */
ASSERT(cachenode->wholevariable); /* by construction */
} else if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
state = FETCHWHOLE;
} else {/* load using constraints */
if(FLAGSET(dapcomm->controls,NCF_WHOLEVAR))
state = FETCHVAR;
else
state = FETCHPART;
}
ASSERT(state != 0);
switch (state) {
case FETCHWHOLE: {
/* buildcachenode3 will create a new cachenode and
will also fetch the whole corresponding datadds.
*/
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* Use no projections or selections */
fetchconstraint->projections = nclistnew();
fetchconstraint->selections = nclistnew();
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHWHOLE: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
case CACHED: {
} break;
case FETCHVAR: { /* Fetch a complete single variable */
/* Create fetch projection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
/* elide any sequence and string dimensions (dap servers do not allow such). */
ncstat = removepseudodims(fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Convert to a whole variable projection */
dcemakewholeprojection(fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHVAR: fetchprojection: |%s|\n",dumpprojection(fetchprojection));
#endif
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* merged constraint just uses the url constraint selection */
fetchconstraint->selections = dceclonelist(dapcomm->oc.dapconstraint->selections);
/* and the created fetch projection */
fetchconstraint->projections = nclistnew();
nclistpush(fetchconstraint->projections,(void*)fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHVAR: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
/* buildcachenode3 will create a new cachenode and
will also fetch the corresponding datadds.
*/
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
case FETCHPART: {
/* Create fetch projection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
/* elide any sequence and string dimensions (dap servers do not allow such). */
ncstat = removepseudodims(fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Shift the varaprojection for simple walk */
dcefree((DCEnode*)walkprojection) ; /* reclaim any existing walkprojection */
walkprojection = (DCEprojection*)dceclone((DCEnode*)varaprojection);
dapshiftprojection(walkprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHPART: fetchprojection: |%s|\n",dumpprojection(fetchprojection));
#endif
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* merged constraint just uses the url constraint selection */
fetchconstraint->selections = dceclonelist(dapcomm->oc.dapconstraint->selections);
/* and the created fetch projection */
fetchconstraint->projections = nclistnew();
nclistpush(fetchconstraint->projections,(void*)fetchprojection);
#ifdef DEBUG
fprintf(stderr,"getvarx: FETCHPART: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
/* buildcachenode3 will create a new cachenode and
will also fetch the corresponding datadds.
*/
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
} break;
default: PANIC1("unknown fetch state: %d\n",state);
}
ASSERT(cachenode != NULL);
#ifdef DEBUG
fprintf(stderr,"cache.datadds=%s\n",dumptree(cachenode->datadds));
#endif
/* attach DATADDS to (constrained) DDS */
unattach34(dapcomm->cdf.ddsroot);
ncstat = attachsubset34(cachenode->datadds,dapcomm->cdf.ddsroot);
if(ncstat) goto fail;
/* Fix up varainfo to use the cache */
varainfo->cache = cachenode;
cachenode = NULL;
varainfo->varaprojection = walkprojection;
walkprojection = NULL;
/* Get the var correlate from the datadds */
target = varainfo->target;
xtarget = target->attachment;
if(xtarget == NULL)
{THROWCHK(ncstat=NC_ENODATA); goto fail;}
/* Switch to datadds tree space*/
varainfo->target = xtarget;
save = (DCEnode*)varaprojection;
ncstat = moveto(dapcomm,varainfo,varainfo->cache->datadds,data);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
nclistfree(vars);
dcefree((DCEnode*)varaprojection);
dcefree((DCEnode*)fetchconstraint);
freegetvara(varainfo);
fail:
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
static NCerror
movetor(NCDAPCOMMON* nccomm,
OCdatanode currentcontent,
NClist* path,
int depth, /* depth is position in segment list*/
Getvara* xgetvar,
size_t dimindex, /* dimindex is position in xgetvar->slices*/
struct NCMEMORY* memory,
NClist* segments)
{
OCerror ocstat = OC_NOERR;
NCerror ncstat = NC_NOERR;
OClink conn = nccomm->oc.conn;
CDFnode* xnode = (CDFnode*)nclistget(path,depth);
OCdatanode reccontent = NULL;
OCdatanode dimcontent = NULL;
OCdatanode fieldcontent = NULL;
Dapodometer* odom = NULL;
int hasstringdim = 0;
DCEsegment* segment;
OCDT mode;
/* Note that we use depth-1 because the path contains the DATASET
but the segment list does not */
segment = (DCEsegment*)nclistget(segments,depth-1); /*may be NULL*/
if(xnode->etype == NC_STRING || xnode->etype == NC_URL) hasstringdim = 1;
/* Get the mode */
mode = oc_data_mode(conn,currentcontent);
#ifdef DEBUG2
fprintf(stderr,"moveto: nctype=%d depth=%d dimindex=%d mode=%s",
xnode->nctype, depth,dimindex,oc_data_modestring(mode));
fprintf(stderr," segment=%s hasstringdim=%d\n",
dcetostring((DCEnode*)segment),hasstringdim);
#endif
switch (xnode->nctype) {
#if 0
#define OCDT_FIELD ((OCDT)(1)) /* field of a container */
#define OCDT_ELEMENT ((OCDT)(2)) /* element of a structure array */
#define OCDT_RECORD ((OCDT)(4)) /* record of a sequence */
#define OCDT_ARRAY ((OCDT)(8)) /* is structure array */
#define OCDT_SEQUENCE ((OCDT)(16)) /* is sequence */
#define OCDT_ATOMIC ((OCDT)(32)) /* is atomic leaf */
#endif
default:
goto done;
case NC_Grid:
case NC_Dataset:
case NC_Structure:
/* Separate out the case where structure is dimensioned */
if(oc_data_indexable(conn,currentcontent)) {
/* => dimensioned structure */
/* The current segment should reflect the
proper parts of the nc_get_vara argument
*/
/* Create odometer for this structure's part of the projection */
odom = dapodom_fromsegment(segment,0,segment->rank);
while(dapodom_more(odom)) {
/* Compute which instance to move to*/
ocstat = oc_data_ithelement(conn,currentcontent,
odom->index,&dimcontent);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
ASSERT(oc_data_indexed(conn,dimcontent));
ncstat = movetor(nccomm,dimcontent,
path,depth,/*keep same depth*/
xgetvar,dimindex+segment->rank,
memory,segments);
dapodom_next(odom);
}
dapodom_free(odom);
} else {/* scalar instance */
ncstat = movetofield(nccomm,currentcontent,path,depth,xgetvar,dimindex,memory,segments);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
} break;
case NC_Sequence:
if(fIsSet(mode,OCDT_SEQUENCE)) {
ASSERT((xnode->attachment != NULL));
ASSERT((segment != NULL));
ASSERT((segment->rank == 1));
/* Build an odometer for walking the sequence,
however, watch out
for the case when the user set a limit and that limit
is not actually reached in this request.
*/
/* By construction, this sequence represents the first
(and only) dimension of this segment */
odom = dapodom_fromsegment(segment,0,1);
while(dapodom_more(odom)) {
size_t recordindex = dapodom_count(odom);
ocstat = oc_data_ithrecord(conn,currentcontent,
recordindex,&reccontent);
if(ocstat != OC_NOERR) {
if(ocstat == OC_EINDEX)
ocstat = OC_EINVALCOORDS;
THROWCHK(ocstat); goto done;
}
ncstat = movetor(nccomm,reccontent,
path,depth,
xgetvar,dimindex+1,
memory,segments);
if(ncstat != OC_NOERR) {THROWCHK(ncstat); goto done;}
dapodom_next(odom);
}
} else if(fIsSet(mode,OCDT_RECORD)) {
/* Treat like structure */
/* currentcontent points to the record instance */
ncstat = movetofield(nccomm,currentcontent,path,depth,xgetvar,dimindex,memory,segments);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
}
break;
case NC_Atomic:
if(hasstringdim)
ncstat = extractstring(nccomm, xgetvar, xnode, segment, dimindex, conn, currentcontent, memory);
else
ncstat = extract(nccomm, xgetvar, xnode, segment, dimindex, conn, currentcontent, memory);
break;
}
done:
oc_data_free(conn,dimcontent);
oc_data_free(conn,fieldcontent);
oc_data_free(conn,reccontent);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
/* See ncd4dispatch.c for other version */
int
NCCR_open(const char * path, int mode,
int basepe, size_t *chunksizehintp,
int useparallel, void* mpidata,
NC_Dispatch* dispatch, NC** ncpp)
{
NCerror ncstat = NC_NOERR;
NC_URL* tmpurl;
NCCR* nccr = NULL; /* reuse the ncdap3 structure*/
NCCDMR* cdmr = NULL;
NC_HDF5_FILE_INFO_T* h5 = NULL;
NC_GRP_INFO_T *grp = NULL;
int ncid = -1;
int fd;
char* tmpname = NULL;
NClist* shows;
bytes_t buf;
long filetime;
ast_runtime* rt = NULL;
ast_err aststat = AST_NOERR;
Header* hdr = NULL;
LOG((1, "nc_open_file: path %s mode %d", path, mode));
if(nc_urlparse(path,&tmpurl) != NC_NOERR) PANIC("libcdmr: non-url path");
nc_urlfree(tmpurl); /* no longer needed */
/* Check for legal mode flags */
if((mode & NC_WRITE) != 0) ncstat = NC_EINVAL;
else if(mode & (NC_WRITE|NC_CLOBBER)) ncstat = NC_EPERM;
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
mode = (mode & ~(NC_MPIIO | NC_MPIPOSIX));
/* Despite the above check, we want the file to be initially writable */
mode |= (NC_WRITE|NC_CLOBBER);
/* Use NCCR code to establish a pseudo file */
tmpname = nulldup(PSEUDOFILE);
fd = mkstemp(tmpname);
if(fd < 0) {THROWCHK(errno); goto done;}
/* Now, use the file to create the hdf5 file */
ncstat = NC4_create(tmpname,NC_NETCDF4|NC_CLOBBER,
0,0,NULL,0,NULL,dispatch,(NC**)&nccr);
ncid = nccr->info.ext_ncid;
/* unlink the temp file so it will automatically be reclaimed */
unlink(tmpname);
free(tmpname);
/* Avoid fill */
dispatch->set_fill(ncid,NC_NOFILL,NULL);
if(ncstat)
{THROWCHK(ncstat); goto done;}
/* Find our metadata for this file. */
ncstat = nc4_find_nc_grp_h5(ncid, (NC_FILE_INFO_T**)&nccr, &grp, &h5);
if(ncstat)
{THROWCHK(ncstat); goto done;}
/* Setup tentative NCCR state*/
nccr->info.dispatch = dispatch;
cdmr = (NCCDMR*)calloc(1,sizeof(NCCDMR));
if(cdmr == NULL) {ncstat = NC_ENOMEM; goto done;}
nccr->cdmr = cdmr;
nccr->cdmr->controller = (NC*)nccr;
nccr->cdmr->urltext = nulldup(path);
nc_urlparse(nccr->cdmr->urltext,&nccr->cdmr->url);
/* Create the curl connection (does not make the server connection)*/
ncstat = nccr_curlopen(&nccr->cdmr->curl.curl);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
shows = nc_urllookup(nccr->cdmr->url,"show");
if(nc_urllookupvalue(shows,"fetch"))
nccr->cdmr->controls |= SHOWFETCH;
/* Parse the projection */
/* fetch meta data */
buf = bytes_t_null;
NCbytes* completeurl = ncbytesnew();
ncbytescat(completeurl,nccr->cdmr->url->url);
ncbytescat(completeurl,"?req=header");
ncstat = nccr_fetchurl(nccr->cdmr->curl.curl,ncbytescontents(completeurl),
&buf,&filetime);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Parse the meta data */
ncstat = nccr_decodeheader(&buf,&hdr);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
if(buf.bytes != NULL) free(buf.bytes);
/* Compute various things about the Header tree */
/* Collect all nodes and fill in the CRnode part*/
cdmr->nodeset = nclistnew();
ncstat = nccr_walk_Header(hdr,cdmr->nodeset);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Compute the pathnames */
ncstat = nccr_compute_pathnames(cdmr->nodeset);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Map dimension references to matching declaration */
ncstat = nccr_map_dimensions(cdmr->nodeset);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Replace dimension references with matching declaration */
nccr_deref_dimensions(cdmr->nodeset);
/* Elide any variables not in the url projection */
ncstat = nccr_elide_vars(cdmr);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* build the meta data */
ncstat = nccr_buildnc(nccr,hdr);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Mark as no longer indef and no longer writable*/
h5->flags &= ~(NC_INDEF);
h5->no_write = 1;
done:
if(aststat != AST_NOERR) {ncstat = nccr_cvtasterr(aststat);}
if(ncstat) {
if(nccr != NULL) {
int ncid = nccr->info.ext_ncid;
NCCR_abort(ncid);
}
} else {
if(ncpp) *ncpp = (NC*)nccr;
}
return THROW(ncstat);
}
/* and any necessary pseudo-dimensions for string types*/
static NCerror
buildvars(NCDAPCOMMON* dapcomm)
{
int i,j;
NCerror ncstat = NC_NOERR;
int varid;
NClist* varnodes = dapcomm->cdf.varnodes;
NC* drno = dapcomm->controller;
char* definename;
ASSERT((varnodes != NULL));
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* var = (CDFnode*)nclistget(varnodes,i);
int dimids[NC_MAX_VAR_DIMS];
unsigned int ncrank;
NClist* vardims = NULL;
if(!var->visible) continue;
if(var->array.basevar != NULL) continue;
#ifdef DEBUG1
fprintf(stderr,"buildvars.candidate=|%s|\n",var->ncfullname);
#endif
vardims = var->array.dimsetall;
ncrank = nclistlength(vardims);
if(ncrank > 0) {
for(j=0;j<ncrank;j++) {
CDFnode* dim = (CDFnode*)nclistget(vardims,j);
dimids[j] = dim->ncid;
}
}
definename = getdefinename(var);
#ifdef DEBUG1
fprintf(stderr,"define: var: %s/%s",
definename,var->ocname);
if(ncrank > 0) {
int k;
for(k=0;k<ncrank;k++) {
CDFnode* dim = (CDFnode*)nclistget(vardims,k);
fprintf(stderr,"[%ld]",dim->dim.declsize);
}
}
fprintf(stderr,"\n");
#endif
ncstat = nc_def_var(drno->substrate,
definename,
var->externaltype,
ncrank,
(ncrank==0?NULL:dimids),
&varid);
nullfree(definename);
if(ncstat != NC_NOERR) {
THROWCHK(ncstat);
goto done;
}
var->ncid = varid;
if(var->attributes != NULL) {
for(j=0;j<nclistlength(var->attributes);j++) {
NCattribute* att = (NCattribute*)nclistget(var->attributes,j);
ncstat = buildattribute3a(dapcomm,att,var->etype,varid);
if(ncstat != NC_NOERR) goto done;
}
}
/* Tag the variable with its DAP path */
if(paramcheck34(dapcomm,"show","projection"))
showprojection3(dapcomm,var);
}
done:
return THROW(ncstat);
}
OCerror
ocfetch(OCstate* state, const char* constraint, OCdxd kind, OCnode** rootp)
{
OCtree* tree = NULL;
OCnode* root = NULL;
OCerror stat = OC_NOERR;
tree = (OCtree*)ocmalloc(sizeof(OCtree));
MEMCHECK(tree,OC_ENOMEM);
memset((void*)tree,0,sizeof(OCtree));
tree->dxdclass = kind;
tree->state = state;
tree->constraint = constraintescape(constraint);
if(tree->constraint == NULL)
tree->constraint = nulldup(constraint);
ocbytesclear(state->packet);
switch (kind) {
case OCDAS:
stat = readDAS(state,tree);
if(stat == OC_NOERR) {
tree->text = ocbytesdup(state->packet);
if(tree->text == NULL) stat = OC_EDAS;
}
break;
case OCDDS:
stat = readDDS(state,tree);
if(stat == OC_NOERR) {
tree->text = ocbytesdup(state->packet);
if(tree->text == NULL) stat = OC_EDDS;
}
break;
case OCDATADDS:
#ifdef OC_DISK_STORAGE
/* Create the datadds file immediately
so that DRNO can reference it*/
/* Make the tmp file*/
stat = createtempfile(state,tree);
if(stat) {THROWCHK(stat); goto unwind;}
stat = readDATADDS(state,tree);
if(stat == OC_NOERR) {
/* Separate the DDS from data and return the dds;
will modify packet */
stat = ocextractdds(state,tree);
}
#else
stat = readDATADDS(state,tree);
if(stat == OC_NOERR) {
/* Separate the DDS from data*/
stat = ocextractdds(state,tree);
tree->data.xdrdata = ocbytesdup(state->packet);
}
#endif
break;
}
if(stat != OC_NOERR) {
/* Obtain any http code */
state->error.httpcode = ocfetchhttpcode(state->curl);
if(state->error.httpcode >= 400) {
oc_log(LOGWARN,"oc_open: Could not read url; http error = %l",state->error.httpcode);
} else {
oc_log(LOGWARN,"oc_open: Could not read url");
}
return THROW(stat);
}
tree->nodes = NULL;
stat = DAPparse(state,tree,tree->text);
/* Check and report on an error return from the server */
if(stat == OC_EDAPSVC && state->error.code != NULL) {
oc_log(LOGERR,"oc_open: server error retrieving url: code=%s message=\"%s\"",
state->error.code,
(state->error.message?state->error.message:""));
}
if(stat) {THROWCHK(stat); goto unwind;}
root = tree->root;
/* make sure */
tree->root = root;
root->tree = tree;
/* Verify the parse */
switch (kind) {
case OCDAS:
if(root->octype != OC_Attributeset)
{THROWCHK(stat=OC_EDAS); goto unwind;}
break;
case OCDDS:
if(root->octype != OC_Dataset)
{THROWCHK(stat=OC_EDDS); goto unwind;}
break;
case OCDATADDS:
if(root->octype != OC_Dataset)
{THROWCHK(stat=OC_EDATADDS); goto unwind;}
/* Modify the tree kind */
tree->dxdclass = OCDATADDS;
break;
default: return OC_EINVAL;
}
if(kind != OCDAS) {
/* Process ocnodes to fix various semantic issues*/
computeocsemantics(tree->nodes);
}
/* Process ocnodes to compute name info*/
computeocfullnames(tree->root);
if(kind != OCDAS) {
/* Process ocnodes to compute sizes when uniform in size*/
ocsetsize(tree->root);
}
if(kind == OCDATADDS) {
tree->data.xdrs = (XDR*)ocmalloc(sizeof(XDR));
MEMCHECK(tree->data.xdrs,OC_ENOMEM);
#ifdef OC_DISK_STORAGE
ocxdrstdio_create(tree->data.xdrs,tree->data.file,XDR_DECODE);
#else
xdrmem_create(tree->data.xdrs,tree->data.xdrdata,tree->data.datasize,XDR_DECODE);
#endif
if(!xdr_setpos(tree->data.xdrs,tree->data.bod)) return xdrerror();
}
#ifdef OC_DISK_STORAGE
if(ocdebug == 0 && tree->data.filename != NULL) {
unlink(tree->data.filename);
}
#endif
/* Put root into the state->trees list */
oclistpush(state->trees,(ocelem)root);
if(rootp) *rootp = root;
return stat;
unwind:
ocfreetree(tree);
return THROW(stat);
}
/* Skip arbitrary dimensioned instance; handles dimensioning.*/
int
ocskip(OCnode* node, XDR* xdrs)
{
unsigned int i,j,rank;
int stat = OC_NOERR;
unsigned int xdrcount;
unsigned int len;
switch (node->octype) {
case OC_Primitive:
/* handle non-uniform types separately*/
if(node->etype == OC_String || node->etype == OC_URL) {
rank = node->array.rank;
xdrcount = 1;
if(rank > 0 && !xdr_u_int(xdrs,&xdrcount)) return xdrerror();
len = xdrcount;
for(i=0;i<xdrcount;i++) {
if(!xdr_u_int(xdrs,&len)) return xdrerror();
if(!xdr_skip(xdrs,len)) return xdrerror();
}
} else { /* uniform => do a direct skip*/
OCASSERT((node->dap.arraysize > 0
&& node->dap.instancesize > 0));
if(node->array.rank > 0) {
if(!xdr_skip(xdrs,node->dap.arraysize)) return xdrerror();
} else {
if(!xdr_skip(xdrs,node->dap.instancesize)) return xdrerror();
}
}
break;
case OC_Grid:
OCASSERT((node->array.rank == 0));
if(node->dap.instancesize > 0) { /* do a direct skip*/
if(!xdr_skip(xdrs,node->dap.arraysize)) return xdrerror();
break;
} else { /* Non-uniform size*/
/* Walk array and maps*/
for(j=0;j<oclistlength(node->subnodes);j++) {
OCnode* field = (OCnode*)oclistget(node->subnodes,j);
stat = ocskip(field,xdrs);
if(stat != OC_NOERR) {THROWCHK(stat); break;}
}
if(stat != OC_NOERR) {THROWCHK(stat); break;}
}
break;
case OC_Dataset:
OCASSERT((node->array.rank == 0));
/* fall-thru*/
case OC_Structure:
if(node->dap.instancesize > 0) { /* do a direct skip*/
if(!xdr_skip(xdrs,node->dap.arraysize)) return xdrerror();
} else {
/* Non-uniform size, we have to skip element by element*/
rank = node->array.rank;
xdrcount = 1;
if(rank > 0 && !xdr_u_int(xdrs,&xdrcount)) return xdrerror();
for(i=0;i<xdrcount;i++) { /* skip element by element*/
/* Walk each structure field*/
for(j=0;j<oclistlength(node->subnodes);j++) {
OCnode* field = (OCnode*)oclistget(node->subnodes,j);
stat = ocskip(field,xdrs);
if(stat != OC_NOERR) {THROWCHK(stat); break;}
}
if(stat != OC_NOERR) {THROWCHK(stat); break;}
}
}
break;
case OC_Sequence: /* not uniform, so walk record by record*/
OCASSERT((node->array.rank == 0));
for(;;) {
/* pick up the sequence record begin marker*/
char tmp[sizeof(unsigned int)];
/* extract the tag byte*/
if(!xdr_opaque(xdrs,tmp,sizeof(tmp))) return xdrerror();
if(tmp[0] == StartOfoclist) {
/* Walk each member field*/
for(j=0;j<oclistlength(node->subnodes);j++) {
OCnode* member = (OCnode*)oclistget(node->subnodes,j);
stat = ocskip(member,xdrs);
if(stat != OC_NOERR) {THROWCHK(stat); break;}
}
} else if(tmp[0] == EndOfoclist) {
break; /* we are done with the this sequence instance*/
} else {
oc_log(LOGERR,"missing/invalid begin/end record marker\n");
stat = OC_EINVALCOORDS;
{THROWCHK(stat); break;}
}
if(stat != OC_NOERR) {THROWCHK(stat); break;}
}
break;
default:
OCPANIC1("oc_move: encountered unexpected node type: %x",node->octype);
break;
}
return THROW(stat);
}
static NCerror
movetor(NCDAPCOMMON* nccomm,
OCdata currentcontent,
NClist* path,
int depth, /* depth is position in segment list*/
Getvara* xgetvar,
int dimindex, /* dimindex is position in xgetvar->slices*/
struct NCMEMORY* memory,
NClist* segments)
{
int i;
OCerror ocstat = OC_NOERR;
NCerror ncstat = NC_NOERR;
size_t fieldindex,gridindex,rank;
OCconnection conn = nccomm->oc.conn;
CDFnode* xnode = (CDFnode*)nclistget(path,depth);
OCdata reccontent = OCNULL;
OCdata dimcontent = OCNULL;
OCdata fieldcontent = OCNULL;
Dapodometer* odom = OCNULL;
OCmode currentmode = OCNULLMODE;
CDFnode* xnext;
int hasstringdim = 0;
size_t dimoffset;
DCEsegment* segment;
int newdepth;
int caching = FLAGSET(nccomm->controls,NCF_CACHE);
int unconstrainable = FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE);
/* Note that we use depth-1 because the path contains the DATASET
but the segment list does not */
segment = (DCEsegment*)nclistget(segments,depth-1); /*may be NULL*/
if(xnode->etype == NC_STRING || xnode->etype == NC_URL) hasstringdim = 1;
ocstat = oc_data_mode(conn,currentcontent,¤tmode);
#ifdef DEBUG2
fprintf(stderr,"moveto: nctype=%d currentmode=%d depth=%d dimindex=%d",
xnode->nctype, currentmode, depth,dimindex);
fprintf(stderr," segment=%s hasstringdim=%d\n",
dcetostring((DCEnode*)segment),hasstringdim);
#endif
/* Switch on the combination of nctype and mode */
#define CASE(nc1,nc2) (nc1*1024+nc2)
/* This must be consistent with the oc mode transition function */
switch (CASE(xnode->nctype,currentmode)) {
default:
PANIC2("Illegal combination: nctype=%d mode=%d",
(int)xnode->nctype,(int)currentmode);
break;
case CASE(NC_Sequence,OCFIELDMODE):
case CASE(NC_Dataset,OCFIELDMODE):
case CASE(NC_Grid,OCFIELDMODE):
case CASE(NC_Structure,OCFIELDMODE):
/* currentcontent points to the grid/dataset/structure instance */
xnext = (CDFnode*)nclistget(path,depth+1);
ASSERT((xnext != NULL));
fieldindex = findfield(xnode,xnext);
/* If the next node is a virtual node, then
we need to effectively
ignore it and use the appropriate subnode.
If the next node is a structuregrid node, then
use it as is.
*/
if(xnext->virtual) {
CDFnode* xgrid = xnext;
xnext = (CDFnode*)nclistget(path,depth+2); /* real node */
gridindex = fieldindex;
fieldindex = findfield(xgrid,xnext);
fieldindex += gridindex;
newdepth = depth+2;
} else {
newdepth = depth+1;
}
fieldcontent = oc_data_new(conn);
ocstat = oc_data_ith(conn,currentcontent,fieldindex,fieldcontent);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;}
ncstat = movetor(nccomm,fieldcontent,
path,newdepth,xgetvar,dimindex,memory,
segments);
break;
case CASE(NC_Sequence,OCARRAYMODE): /* will actually always be scalar, but will have
rank == 1 to account for the sequence dim */
case CASE(NC_Grid,OCARRAYMODE): /* will actually always be scalar */
case CASE(NC_Structure,OCARRAYMODE):
/* figure out which slices refer to this node:
dimindex upto dimindex+rank; */
ASSERT((segment != NULL));
rank = segment->rank;
if(xnode->nctype == NC_Sequence)
rank--; /* ignore the sequence dim */
if(rank == 0) {
odom = newdapodometer1(1);
} else if(caching || unconstrainable) {
odom = newdapodometer(segment->slices,0,rank);
} else { /*Since vara was projected out, build a simple odometer*/
odom = newsimpledapodometer(segment,rank);
}
while(dapodometermore(odom)) {
OCmode mode;
/* Compute which instance to move to*/
dimoffset = dapodometercount(odom);
dimcontent = oc_data_new(conn);
ocstat = oc_data_ith(conn,currentcontent,dimoffset,dimcontent);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;}
ocstat = oc_data_mode(conn,dimcontent,&mode);
ASSERT((mode == OCFIELDMODE
|| (mode == OCSEQUENCEMODE && xnode->nctype == NC_Sequence)));
ncstat = movetor(nccomm,dimcontent,
path,depth,
xgetvar,dimindex+rank,
memory,segments);
dapodometerincr(odom);
}
freedapodometer(odom);
break;
case CASE(NC_Sequence,OCSEQUENCEMODE): {
DCEslice* uslice;
ASSERT((segment != NULL));
/* Get and check the corresponding sequence dimension from DDS */
ASSERT((xnode->attachment != NULL));
/* use uslice to walk the sequence; however, watch out
for the case when the user set a limit and that limit
is not actually reached in this request.
*/
/* By construction, this sequence represents the first
(and only) dimension of this segment */
uslice = &segment->slices[0];
reccontent = oc_data_new(conn);
for(i=uslice->first;i<uslice->stop;i+=uslice->stride) {
OCmode eos;
ocstat = oc_data_ith(conn,currentcontent,i,reccontent);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;}
ocstat = oc_data_mode(conn,reccontent,&eos);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto fail;}
if(eos == OCNULLMODE) {
/* We asked for too much */
ncstat = THROW(NC_EINVALCOORDS);
goto fail;
}
ncstat = movetor(nccomm,reccontent,
path,depth,
xgetvar,dimindex+1,
memory,segments);
if(ncstat != OC_NOERR) {THROWCHK(ncstat); goto fail;}
}
} break;
case CASE(NC_Primitive,OCPRIMITIVEMODE):
if(hasstringdim)
ncstat = extractstring(nccomm, xgetvar, xnode, segment, conn, currentcontent, memory);
else
ncstat = extract(nccomm, xgetvar, xnode, segment, conn, currentcontent, memory);
break;
}
static NCerror
builddims(NCDAPCOMMON* dapcomm)
{
int i;
NCerror ncstat = NC_NOERR;
int dimid;
NClist* dimset = NULL;
NC* drno = dapcomm->controller;
NC* ncsub;
char* definename;
/* collect all dimensions from variables */
dimset = dapcomm->cdf.dimnodes;
/* Sort by fullname just for the fun of it */
for(;;) {
int last = nclistlength(dimset) - 1;
int swap = 0;
for(i=0;i<last;i++) {
CDFnode* dim1 = (CDFnode*)nclistget(dimset,i);
CDFnode* dim2 = (CDFnode*)nclistget(dimset,i+1);
if(strcmp(dim1->ncfullname,dim2->ncfullname) > 0) {
nclistset(dimset,i,(ncelem)dim2);
nclistset(dimset,i+1,(ncelem)dim1);
swap = 1;
break;
}
}
if(!swap) break;
}
/* Define unlimited only if needed */
if(dapcomm->cdf.recorddim != NULL) {
CDFnode* unlimited = dapcomm->cdf.recorddim;
definename = getdefinename(unlimited);
ncstat = nc_def_dim(drno->substrate,
definename,
NC_UNLIMITED,
&unlimited->ncid);
nullfree(definename);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* get the id for the substrate */
ncstat = NC_check_id(drno->substrate,&ncsub);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Set the effective size of UNLIMITED;
note that this cannot be done thru the normal API.*/
NC_set_numrecs(ncsub,unlimited->dim.declsize);
}
for(i=0;i<nclistlength(dimset);i++) {
CDFnode* dim = (CDFnode*)nclistget(dimset,i);
if(dim->dim.basedim != NULL) continue; /* handle below */
if(DIMFLAG(dim,CDFDIMRECORD)) continue; /* defined above */
#ifdef DEBUG1
fprintf(stderr,"define: dim: %s=%ld\n",dim->ncfullname,(long)dim->dim.declsize);
#endif
definename = getdefinename(dim);
ncstat = nc_def_dim(drno->substrate,definename,dim->dim.declsize,&dimid);
if(ncstat != NC_NOERR) {
THROWCHK(ncstat); goto done;
}
nullfree(definename);
dim->ncid = dimid;
}
/* Make all duplicate dims have same dimid as basedim*/
/* (see computecdfdimnames)*/
for(i=0;i<nclistlength(dimset);i++) {
CDFnode* dim = (CDFnode*)nclistget(dimset,i);
if(dim->dim.basedim != NULL) {
dim->ncid = dim->dim.basedim->ncid;
}
}
done:
nclistfree(dimset);
return THROW(ncstat);
}
NCerror
buildcachenode34(NCDAPCOMMON* nccomm,
DCEconstraint* constraint,
NClist* varlist,
NCcachenode** cachep,
NCFLAGS flags)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
OClink conn = nccomm->oc.conn;
OCddsnode ocroot = NULL;
CDFnode* dxdroot = NULL;
NCcachenode* cachenode = NULL;
char* ce = NULL;
int isprefetch = 0;
if((flags & NCF_PREFETCH) != 0)
isprefetch = 1;
if((flags & NCF_PREFETCH_ALL) == 0)
ce = buildconstraintstring3(constraint);
ocstat = dap_fetch(nccomm,conn,ce,OCDATADDS,&ocroot);
nullfree(ce);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
ncstat = buildcdftree34(nccomm,ocroot,OCDATA,&dxdroot);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* re-struct*/
if(!FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
ncstat = restruct3(dxdroot,nccomm->cdf.ddsroot,constraint->projections);
if(ncstat) {THROWCHK(ncstat); goto done;}
}
/* create the cache node */
cachenode = createnccachenode();
cachenode->isprefetch = isprefetch;
cachenode->vars = nclistclone(varlist);
cachenode->datadds = dxdroot;
/* Give the constraint over to the cachenode */
cachenode->constraint = constraint;
constraint = NULL;
cachenode->wholevariable = iscacheableconstraint(cachenode->constraint);
/* save the root content*/
cachenode->ocroot = ocroot;
ocstat = oc_data_getroot(conn,ocroot,&cachenode->content);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
/* capture the packet size */
ocstat = oc_raw_xdrsize(conn,ocroot,&cachenode->xdrsize);
if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;}
#ifdef DEBUG
fprintf(stderr,"buildcachenode: new cache node: %s\n",
dumpcachenode(cachenode));
#endif
/* Insert into the cache. If not caching, then
remove any previous cache node
*/
if(!isprefetch) {
NCcache* cache = nccomm->cdf.cache;
if(cache->nodes == NULL) cache->nodes = nclistnew();
/* remove cache nodes to get below the max cache size */
while(cache->cachesize + cachenode->xdrsize > cache->cachelimit
&& nclistlength(cache->nodes) > 0) {
NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0);
#ifdef DEBUG
fprintf(stderr,"buildcachenode: purge cache node: %s\n",
dumpcachenode(cachenode));
#endif
cache->cachesize -= node->xdrsize;
freenccachenode(nccomm,node);
}
/* Remove cache nodes to get below the max cache count */
/* If not caching, then cachecount should be 0 */
while(nclistlength(cache->nodes) > cache->cachecount) {
NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0);
#ifdef DEBUG
fprintf(stderr,"buildcachenode: count purge cache node: %s\n",
dumpcachenode(node));
#endif
cache->cachesize -= node->xdrsize;
freenccachenode(nccomm,node);
}
nclistpush(nccomm->cdf.cache->nodes,(void*)cachenode);
cache->cachesize += cachenode->xdrsize;
}
#ifdef DEBUG
fprintf(stderr,"buildcachenode: %s\n",dumpcachenode(cachenode));
#endif
done:
if(constraint != NULL) dcefree((DCEnode*)constraint);
if(cachep) *cachep = cachenode;
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
if(ncstat) {
freecdfroot34(dxdroot);
freenccachenode(nccomm,cachenode);
}
return THROW(ncstat);
}
/* See ncd3dispatch.c for other version */
int
NCD3_open(const char * path, int mode,
int basepe, size_t *chunksizehintp,
int useparallel, void* mpidata,
NC_Dispatch* dispatch, NC** ncpp)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
NC* drno = NULL;
NCDAPCOMMON* dapcomm = NULL;
const char* value;
char* tmpname = NULL;
if(!nc3dinitialized) nc3dinitialize();
if(path == NULL)
return NC_EDAPURL;
if(dispatch == NULL) PANIC("NC3D_open: no dispatch table");
/* Setup our NC and NCDAPCOMMON state*/
drno = (NC*)calloc(1,sizeof(NC));
if(drno == NULL) {ncstat = NC_ENOMEM; goto done;}
/* compute an ncid */
ncstat = add_to_NCList(drno);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
dapcomm = (NCDAPCOMMON*)calloc(1,sizeof(NCDAPCOMMON));
if(dapcomm == NULL) {ncstat = NC_ENOMEM; goto done;}
drno->dispatch = dispatch;
drno->dispatchdata = dapcomm;
dapcomm->controller = (NC*)drno;
dapcomm->cdf.separator = ".";
dapcomm->cdf.smallsizelimit = DFALTSMALLLIMIT;
dapcomm->cdf.cache = createnccache();
#ifdef HAVE_GETRLIMIT
{ struct rlimit rl;
if(getrlimit(RLIMIT_NOFILE, &rl) >= 0) {
dapcomm->cdf.cache->cachecount = (size_t)(rl.rlim_cur / 2);
}
}
#endif
#ifdef OCCOMPILEBYDEFAULT
/* set the compile flag by default */
dapcomm->oc.rawurltext = (char*)emalloc(strlen(path)+strlen("[compile]")+1);
strcpy(dapcomm->oc.rawurltext,"[compile]");
strcat(dapcomm->oc.rawurltext, path);
#else
dapcomm->oc.rawurltext = strdup(path);
#endif
nc_uriparse(dapcomm->oc.rawurltext,&dapcomm->oc.url);
/* parse the client parameters */
nc_uridecodeparams(dapcomm->oc.url);
if(!constrainable34(dapcomm->oc.url))
SETFLAG(dapcomm->controls,NCF_UNCONSTRAINABLE);
/* Use libsrc code for storing metadata */
tmpname = nulldup(PSEUDOFILE);
/* Now, use the file to create the netcdf file */
if(sizeof(size_t) == sizeof(unsigned int))
ncstat = nc_create(tmpname,NC_CLOBBER,&drno->substrate);
else
ncstat = nc_create(tmpname,NC_CLOBBER|NC_64BIT_OFFSET,&drno->substrate);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* free the filename so it will automatically go away*/
unlink(tmpname);
nullfree(tmpname);
/* Avoid fill */
nc_set_fill(drno->substrate,NC_NOFILL,NULL);
dapcomm->oc.dapconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
dapcomm->oc.dapconstraint->projections = nclistnew();
dapcomm->oc.dapconstraint->selections = nclistnew();
/* Parse constraints to make sure they are syntactically correct */
ncstat = parsedapconstraints(dapcomm,dapcomm->oc.url->constraint,dapcomm->oc.dapconstraint);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Complain if we are unconstrainable but have constraints */
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
if(dapcomm->oc.url->constraint != NULL
&& strlen(dapcomm->oc.url->constraint) > 0) {
nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s",
dapcomm->oc.url->constraint);
}
}
/* Construct a url for oc minus any parameters */
dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,
(NC_URIALL ^ NC_URICONSTRAINTS));
/* Pass to OC */
ocstat = oc_open(dapcomm->oc.urltext,&dapcomm->oc.conn);
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
nullfree(dapcomm->oc.urltext); /* clean up */
dapcomm->oc.urltext = NULL;
/* process control client parameters */
applyclientparamcontrols3(dapcomm);
/* Turn on logging; only do this after oc_open*/
if((value = paramvalue34(dapcomm,"log")) != NULL) {
ncloginit();
ncsetlogging(1);
nclogopen(value);
oc_loginit();
oc_setlogging(1);
oc_logopen(value);
}
/* fetch and build the (almost) unconstrained DDS for use as
template */
ncstat = fetchtemplatemetadata3(dapcomm);
if(ncstat != NC_NOERR) goto done;
/* fetch and build the constrained DDS */
ncstat = fetchconstrainedmetadata3(dapcomm);
if(ncstat != NC_NOERR) goto done;
#ifdef DEBUG2
fprintf(stderr,"constrained dds: %s\n",dumptree(dapcomm->cdf.ddsroot));
#endif
/* The following actions are (mostly) WRT to the constrained tree */
/* Accumulate useful nodes sets */
ncstat = computecdfnodesets3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Fix grids */
ncstat = fixgrids3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Locate and mark usable sequences */
ncstat = sequencecheck3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* suppress variables not in usable sequences */
ncstat = suppressunusablevars3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* apply client parameters */
ncstat = applyclientparams34(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Add (as needed) string dimensions*/
ncstat = addstringdims(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
if(nclistlength(dapcomm->cdf.seqnodes) > 0) {
/* Build the sequence related dimensions */
ncstat = defseqdims(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
}
/* Define the dimsetplus and dimsetall lists */
ncstat = definedimsets3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Re-compute the dimension names*/
ncstat = computecdfdimnames34(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Deal with zero size dimensions */
ncstat = fixzerodims3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Attempt to use the DODS_EXTRA info to turn
one of the dimensions into unlimited.
Assume computecdfdimnames34 has already been called.
*/
ncstat = defrecorddim3(dapcomm);
if(ncstat) {THROWCHK(ncstat); goto done;}
if(dapcomm->cdf.recorddimname != NULL
&& nclistlength(dapcomm->cdf.seqnodes) > 0) {
/*nclog(NCLOGWARN,"unlimited dimension specified, but sequences exist in DDS");*/
PANIC("unlimited dimension specified, but sequences exist in DDS");
}
/* Re-compute the var names*/
ncstat = computecdfvarnames3(dapcomm,dapcomm->cdf.ddsroot,dapcomm->cdf.varnodes);
if(ncstat) {THROWCHK(ncstat); goto done;}
/* Transfer data from the unconstrained DDS data to the unconstrained DDS */
ncstat = dimimprint3(dapcomm);
if(ncstat) goto done;
/* Process the constraints to map to the constrained CDF tree */
/* (must follow fixgrids3 */
ncstat = mapconstraints3(dapcomm->oc.dapconstraint,dapcomm->cdf.ddsroot);
if(ncstat != NC_NOERR) goto done;
/* Canonicalize the constraint */
ncstat = fixprojections(dapcomm->oc.dapconstraint->projections);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Fill in segment information */
ncstat = qualifyconstraints3(dapcomm->oc.dapconstraint);
if(ncstat != NC_NOERR) goto done;
/* using the modified constraint, rebuild the constraint string */
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
/* ignore all constraints */
dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,0);
} else {
char* constraintstring = buildconstraintstring3(dapcomm->oc.dapconstraint);
nc_urisetconstraints(dapcomm->oc.url,constraintstring);
nullfree(constraintstring);
dapcomm->oc.urltext = nc_uribuild(dapcomm->oc.url,NULL,NULL,NC_URICONSTRAINTS);
}
#ifdef DEBUG
fprintf(stderr,"ncdap3: final constraint: %s\n",dapcomm->oc.url->constraint);
#endif
/* Estimate the variable sizes */
estimatevarsizes3(dapcomm);
/* Build the meta data */
ncstat = buildncstructures3(dapcomm);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
/* Do any necessary data prefetch */
if(FLAGSET(dapcomm->controls,NCF_PREFETCH)) {
ncstat = prefetchdata3(dapcomm);
if(ncstat != NC_NOERR) {
del_from_NCList((NC*)drno); /* undefine here */
{THROWCHK(ncstat); goto done;}
}
}
{
/* Mark as no longer writable and no longer indef;
requires breaking abstraction */
NC* nc;
ncstat = NC_check_id(drno->substrate, &nc);
/* Mark as no longer writeable */
fClr(nc->nciop->ioflags, NC_WRITE);
/* Mark as no longer indef;
(do NOT use nc_enddef until diskless is working)*/
fSet(nc->flags, NC_INDEF);
}
if(ncpp) *ncpp = (NC*)drno;
return ncstat;
done:
if(drno != NULL) NCD3_abort(drno->ext_ncid);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
/* Compute the set of prefetched data.
Notes:
1. All prefetches are whole variable fetches.
2. If the data set is unconstrainable, we
will prefetch the whole thing
*/
NCerror
prefetchdata3(NCDAPCOMMON* nccomm)
{
int i;
NCFLAGS flags;
NCerror ncstat = NC_NOERR;
NClist* allvars = nccomm->cdf.ddsroot->tree->varnodes;
DCEconstraint* urlconstraint = nccomm->oc.dapconstraint;
NClist* vars = nclistnew();
NCcachenode* cache = NULL;
DCEconstraint* newconstraint = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
} else { /* do no prefetching */
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
} else {
/* pull in those variables previously marked as prefetchable */
for(i=0;i<nclistlength(allvars);i++) {
CDFnode* var = (CDFnode*)nclistget(allvars,i);
/* Most of the important testing was already done */
if(!var->basenode->prefetchable)
continue;
/* Do not attempt to prefetch any variables in the
nc_open url's projection list
*/
if(nclistcontains(nccomm->cdf.projectedvars,(void*)var))
continue;
/* Should be prefetchable */
nclistpush(vars,(void*)var);
if(SHOWFETCH) {
nclog(NCLOGDBG,"prefetch: %s",var->ncfullname);
}
}
}
/* If there are no vars, then do nothing */
if(nclistlength(vars) == 0) {
nccomm->cdf.cache->prefetch = NULL;
goto done;
}
/* Create a single constraint consisting of the projections for the variables;
each projection is whole variable. The selections are passed on as is.
The exception is if we are prefetching everything.
*/
newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
newconstraint->projections = nclistnew();
newconstraint->selections = dceclonelist(urlconstraint->selections);
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
DCEprojection* varprojection;
/* convert var to a projection */
ncstat = dapvar2projection(var,&varprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
nclistpush(newconstraint->projections,(void*)varprojection);
}
if(SHOWFETCH) {
char* s = dumpprojections(newconstraint->projections);
LOG1(NCLOGNOTE,"prefetch.final: %s",s);
nullfree(s);
}
flags = NCF_PREFETCH;
if(nclistlength(allvars) == nclistlength(vars)) flags |= NCF_PREFETCH_ALL;
ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,flags);
newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */
if(ncstat) goto done;
cache->wholevariable = 1; /* All prefetches are whole variable */
/* Make cache node be the prefetch node */
nccomm->cdf.cache->prefetch = cache;
if(SHOWFETCH) {
LOG0(NCLOGNOTE,"prefetch.complete");
}
if(SHOWFETCH) {
char* s = NULL;
/* Log the set of prefetch variables */
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"prefetch.vars: ");
for(i=0;i<nclistlength(vars);i++) {
CDFnode* var = (CDFnode*)nclistget(vars,i);
ncbytescat(buf," ");
s = makecdfpathstring3(var,".");
ncbytescat(buf,s);
nullfree(s);
}
ncbytescat(buf,"\n");
nclog(NCLOGNOTE,"%s",ncbytescontents(buf));
ncbytesfree(buf);
}
done:
nclistfree(vars);
dcefree((DCEnode*)newconstraint);
if(ncstat) freenccachenode(nccomm,cache);
return THROW(ncstat);
}
