str FITSdir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str dir = *(str*)getArgReference(stk, pci, 1);
DIR *dp;
struct dirent *ep;
fitsfile *fptr;
char *s;
int status = 0;
(void)mb;
dp = opendir(dir);
if (dp != NULL) {
char stmt[BUFSIZ];
char fname[BUFSIZ];
s = stmt;
while ((ep = readdir(dp)) != NULL) {
snprintf(fname, BUFSIZ, "%s%s", dir, ep->d_name);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (status == 0) {
snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
msg = SQLstatementIntern(cntxt, &s, "fits.listofdir", TRUE, FALSE);
fits_close_file(fptr, &status);
}
}
(void)closedir(dp);
}else
msg = createException(MAL, "listdir", "Couldn't open the directory");
return msg;
}
str
SQLstatement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str *expr = getArgReference_str(stk, pci, 1);
bit output = TRUE;
(void) mb;
if (pci->argc == 3)
output = *getArgReference_bit(stk, pci, 2);
return SQLstatementIntern(cntxt, expr, "SQLstatement", TRUE, output, NULL);
}
str
SQLcompile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str *ret = getArgReference_str(stk, pci, 0);
str *expr = getArgReference_str(stk, pci, 1);
str msg;
(void) mb;
*ret = NULL;
msg = SQLstatementIntern(cntxt, expr, "SQLcompile", FALSE, FALSE, NULL);
if (msg == MAL_SUCCEED)
*ret = _STRDUP("SQLcompile");
return msg;
}
/*
* Locate a file with SQL commands and execute it. For the time being a 1MB
* file limit is implicitly imposed. If the file can not be located in the
* script library, we assume it is sufficiently self descriptive.
* (Respecting the file system context where the call is executed )
*/
str
SQLinclude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
stream *fd;
bstream *bfd;
str *name = getArgReference_str(stk, pci, 1);
str msg = MAL_SUCCEED, fullname;
str *expr;
mvc *m;
size_t sz;
fullname = MSP_locate_sqlscript(*name, 0);
if (fullname == NULL)
fullname = *name;
fd = open_rastream(fullname);
if (mnstr_errnr(fd) == MNSTR_OPEN_ERROR) {
mnstr_destroy(fd);
throw(MAL, "sql.include", "could not open file: %s\n", *name);
}
sz = getFileSize(fd);
if (sz > (size_t) 1 << 29) {
mnstr_destroy(fd);
throw(MAL, "sql.include", "file %s too large to process", fullname);
}
bfd = bstream_create(fd, sz == 0 ? (size_t) (128 * BLOCK) : sz);
if (bstream_next(bfd) < 0) {
bstream_destroy(bfd);
throw(MAL, "sql.include", "could not read %s\n", *name);
}
expr = &bfd->buf;
msg = SQLstatementIntern(cntxt, expr, "sql.include", TRUE, FALSE, NULL);
bstream_destroy(bfd);
m = ((backend *) cntxt->sqlcontext)->mvc;
if (m->sa)
sa_destroy(m->sa);
m->sa = NULL;
(void) mb;
return msg;
}
str FITSdirpat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str dir = *getArgReference_str(stk, pci, 1);
str pat = *getArgReference_str(stk, pci, 2);
fitsfile *fptr;
char *s;
int status = 0;
glob_t globbuf;
char fulldirectory[BUFSIZ];
size_t j = 0;
(void)mb;
globbuf.gl_offs = 0;
snprintf(fulldirectory, BUFSIZ, "%s%s", dir, pat);
glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf);
/* fprintf(stderr,"#fulldir: %s \nSize: %lu\n",fulldirectory, globbuf.gl_pathc);*/
if (globbuf.gl_pathc == 0)
throw(MAL, "listdir", "Couldn't open the directory or there are no files that match the pattern");
for (j = 0; j < globbuf.gl_pathc; j++) {
char stmt[BUFSIZ];
char fname[BUFSIZ];
s = stmt;
snprintf(fname, BUFSIZ, "%s", globbuf.gl_pathv[j]);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (status == 0) {
snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
msg = SQLstatementIntern(cntxt, &s, "fits.listofdirpat", TRUE, FALSE, NULL);
fits_close_file(fptr, &status);
break;
}
}
return msg;
}
void* RAPIloopback(void *query) {
res_table* output = NULL;
char* querystr = (char*)CHAR(STRING_ELT(query, 0));
char* err = SQLstatementIntern(rapiClient, &querystr, "name", 1, 0, &output);
if (err) { // there was an error
return ScalarString(RSTR(err));
}
if (output) {
int ncols = output->nr_cols;
if (ncols > 0) {
int i;
SEXP retlist, names, varvalue = R_NilValue;
retlist = PROTECT(allocVector(VECSXP, ncols));
names = PROTECT(NEW_STRING(ncols));
for (i = 0; i < ncols; i++) {
BAT *b = BATdescriptor(output->cols[i].b);
if (b == NULL || !(varvalue = bat_to_sexp(b, TYPE_any))) {
UNPROTECT(i + 3);
if (b)
BBPunfix(b->batCacheid);
return ScalarString(RSTR("Conversion error"));
}
BBPunfix(b->batCacheid);
SET_STRING_ELT(names, i, RSTR(output->cols[i].name));
SET_VECTOR_ELT(retlist, i, varvalue);
}
res_table_destroy(output);
SET_NAMES(retlist, names);
UNPROTECT(ncols + 2);
return retlist;
}
res_table_destroy(output);
}
return ScalarLogical(1);
}
str
SQLinitClient(Client c)
{
mvc *m;
str schema;
str msg = MAL_SUCCEED;
backend *be;
bstream *bfd = NULL;
stream *fd = NULL;
static int maybeupgrade = 1;
#ifdef _SQL_SCENARIO_DEBUG
mnstr_printf(GDKout, "#SQLinitClient\n");
#endif
if (SQLinitialized == 0 && (msg = SQLprelude(NULL)) != MAL_SUCCEED)
return msg;
MT_lock_set(&sql_contextLock);
/*
* Based on the initialization return value we can prepare a SQLinit
* string with all information needed to initialize the catalog
* based on the mandatory scripts to be executed.
*/
if (sqlinit) { /* add sqlinit to the fdin stack */
buffer *b = (buffer *) GDKmalloc(sizeof(buffer));
size_t len = strlen(sqlinit);
bstream *fdin;
buffer_init(b, _STRDUP(sqlinit), len);
fdin = bstream_create(buffer_rastream(b, "si"), b->len);
bstream_next(fdin);
MCpushClientInput(c, fdin, 0, "");
}
if (c->sqlcontext == 0) {
m = mvc_create(c->idx, 0, SQLdebug, c->fdin, c->fdout);
global_variables(m, "monetdb", "sys");
if (isAdministrator(c) || strcmp(c->scenario, "msql") == 0) /* console should return everything */
m->reply_size = -1;
be = (void *) backend_create(m, c);
} else {
be = c->sqlcontext;
m = be->mvc;
mvc_reset(m, c->fdin, c->fdout, SQLdebug, NR_GLOBAL_VARS);
backend_reset(be);
}
if (m->session->tr)
reset_functions(m->session->tr);
/* pass through credentials of the user if not console */
schema = monet5_user_set_def_schema(m, c->user);
if (!schema) {
_DELETE(schema);
throw(PERMD, "SQLinitClient", "08004!schema authorization error");
}
_DELETE(schema);
/*expect SQL text first */
be->language = 'S';
/* Set state, this indicates an initialized client scenario */
c->state[MAL_SCENARIO_READER] = c;
c->state[MAL_SCENARIO_PARSER] = c;
c->state[MAL_SCENARIO_OPTIMIZE] = c;
c->sqlcontext = be;
initSQLreferences();
/* initialize the database with predefined SQL functions */
if (SQLnewcatalog == 0) {
/* check whether table sys.systemfunctions exists: if
* it doesn't, this is probably a restart of the
* server after an incomplete initialization */
sql_schema *s = mvc_bind_schema(m, "sys");
sql_table *t = s ? mvc_bind_table(m, s, "systemfunctions") : NULL;
if (t == NULL)
SQLnewcatalog = 1;
}
if (SQLnewcatalog > 0) {
char path[PATHLENGTH];
str fullname;
SQLnewcatalog = 0;
maybeupgrade = 0;
snprintf(path, PATHLENGTH, "createdb");
slash_2_dir_sep(path);
fullname = MSP_locate_sqlscript(path, 1);
if (fullname) {
str filename = fullname;
str p, n;
fprintf(stdout, "# SQL catalog created, loading sql scripts once\n");
do {
p = strchr(filename, PATH_SEP);
if (p)
*p = '\0';
if ((n = strrchr(filename, DIR_SEP)) == NULL) {
n = filename;
} else {
n++;
}
fprintf(stdout, "# loading sql script: %s\n", n);
fd = open_rastream(filename);
if (p)
filename = p + 1;
if (fd) {
size_t sz;
sz = getFileSize(fd);
if (sz > (size_t) 1 << 29) {
mnstr_destroy(fd);
msg = createException(MAL, "createdb", "file %s too large to process", filename);
} else {
bfd = bstream_create(fd, sz == 0 ? (size_t) (128 * BLOCK) : sz);
if (bfd && bstream_next(bfd) >= 0)
msg = SQLstatementIntern(c, &bfd->buf, "sql.init", TRUE, FALSE, NULL);
bstream_destroy(bfd);
}
if (m->sa)
sa_destroy(m->sa);
m->sa = NULL;
if (msg)
p = NULL;
}
} while (p);
GDKfree(fullname);
} else
fprintf(stderr, "!could not read createdb.sql\n");
} else { /* handle upgrades */
if (!m->sa)
m->sa = sa_create();
if (maybeupgrade)
SQLupgrades(c,m);
maybeupgrade = 0;
}
MT_lock_unset(&sql_contextLock);
fflush(stdout);
fflush(stderr);
/* send error from create scripts back to the first client */
if (msg) {
error(c->fdout, msg);
handle_error(m, c->fdout, 0);
sqlcleanup(m, mvc_status(m));
}
return msg;
}
/* import variable given file id and variable name */
str
NCDFimportVariable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch = NULL;
sql_table *tfiles = NULL, *arr_table = NULL;
sql_column *col;
str msg = MAL_SUCCEED, vname = *(str*)getArgReference(stk, pci, 2);
str fname = NULL, dimtype = NULL, aname_sys = NULL;
int fid = *(int*)getArgReference(stk, pci, 1);
int varid, vndims, vnatts, i, j, retval;
char buf[BUFSIZ], *s= buf, aname[256], **dname;
oid rid = oid_nil;
int vdims[NC_MAX_VAR_DIMS];
nc_type vtype;
int ncid; /* dataset id */
size_t dlen;
bat vbatid = 0, *dim_bids;
BAT *vbat = NULL, *dimbat;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
sch = mvc_bind_schema(m, "sys");
if ( !sch )
return createException(MAL, "netcdf.importvar", "Cannot get schema sys\n");
tfiles = mvc_bind_table(m, sch, "netcdf_files");
if (tfiles == NULL)
return createException(MAL, "netcdf.importvar", "Catalog table missing\n");
/* get the name of the attached NetCDF file */
col = mvc_bind_column(m, tfiles, "file_id");
if (col == NULL)
return createException(MAL, "netcdf.importvar", "Could not find \"netcdf_files\".\"file_id\"\n");
rid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
if (rid == oid_nil)
return createException(MAL, "netcdf.importvar", "File %d not in the NetCDF vault\n", fid);
col = mvc_bind_column(m, tfiles, "location");
fname = (str)table_funcs.column_find_value(m->session->tr, col, rid);
/* Open NetCDF file */
if ((retval = nc_open(fname, NC_NOWRITE, &ncid)))
return createException(MAL, "netcdf.importvar", "Cannot open NetCDF file %s: %s",
fname, nc_strerror(retval));
/* Get info for variable vname from NetCDF file */
if ( (retval = nc_inq_varid(ncid, vname, &varid)) )
return createException(MAL, "netcdf.importvar",
"Cannot read variable %s: %s",
vname, nc_strerror(retval));
if ( (retval = nc_inq_var(ncid, varid, vname, &vtype, &vndims, vdims, &vnatts)))
return createException(MAL, "netcdf.importvar",
"Cannot read variable %d : %s",
varid, nc_strerror(retval));
/* compose 'create table' statement in the buffer */
dname = (char **) GDKzalloc( sizeof(char *) * vndims);
for (i = 0; i < vndims; i++)
dname[i] = (char *) GDKzalloc(NC_MAX_NAME + 1);
snprintf(aname, 256, "%s%d", vname, fid);
j = snprintf(buf, BUFSIZ,"create table %s.%s( ", sch->base.name, aname);
for (i = 0; i < vndims; i++){
if ((retval = nc_inq_dim(ncid, vdims[i], dname[i], &dlen)))
return createException(MAL, "netcdf.importvar",
"Cannot read dimension %d : %s",
vdims[i], nc_strerror(retval));
if ( dlen <= (int) GDK_bte_max )
dimtype = "TINYINT";
else if ( dlen <= (int) GDK_sht_max )
dimtype = "SMALLINT";
else
dimtype = "INT";
(void)dlen;
j += snprintf(buf + j, BUFSIZ - j, "%s %s, ", dname[i], dimtype);
}
j += snprintf(buf + j, BUFSIZ - j, "value %s);", NCDF2SQL(vtype));
/* execute 'create table ' */
msg = SQLstatementIntern(cntxt, &s, "netcdf.importvar", TRUE, FALSE, NULL);
if (msg != MAL_SUCCEED)
return msg;
/* load variable data */
dim_bids = (bat *)GDKmalloc(sizeof(bat) * vndims);
msg = NCDFloadVar(&dim_bids, &vbatid, ncid, varid, vtype, vndims, vdims);
if ( msg != MAL_SUCCEED )
return msg;
/* associate columns in the table with loaded variable data */
aname_sys = toLower(aname);
arr_table = mvc_bind_table(m, sch, aname_sys);
if (arr_table == NULL)
return createException(MAL, "netcdf.importvar", "netcdf table %s missing\n", aname_sys);
col = mvc_bind_column(m, arr_table, "value");
if (col == NULL)
return createException(MAL, "netcdf.importvar", "Cannot find column %s.value\n", aname_sys);
vbat = BATdescriptor(vbatid);
store_funcs.append_col(m->session->tr, col, vbat, TYPE_bat);
BBPunfix(vbatid);
BBPdecref(vbatid, 1);
vbat = NULL;
/* associate dimension bats */
for (i = 0; i < vndims; i++){
col = mvc_bind_column(m, arr_table, dname[i]);
if (col == NULL)
return createException(MAL, "netcdf.importvar", "Cannot find column %s.%s\n", aname_sys, dname[i]);
dimbat = BATdescriptor(dim_bids[i]);
store_funcs.append_col(m->session->tr, col, dimbat, TYPE_bat);
BBPunfix(dim_bids[i]); /* phys. ref from BATdescriptor */
BBPdecref(dim_bids[i], 1); /* log. ref. from loadVar */
dimbat = NULL;
}
for (i = 0; i < vndims; i++)
GDKfree(dname[i]);
GDKfree(dname);
GDKfree(dim_bids);
nc_close(ncid);
return msg;
}
str
NCDFattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch = NULL;
sql_table *tfiles = NULL, *tdims = NULL, *tvars = NULL, *tvardim = NULL, *tattrs = NULL;
sql_column *col;
str msg = MAL_SUCCEED;
str fname = *(str*)getArgReference(stk, pci, 1);
char buf[BUFSIZ], *s= buf;
oid fid, rid = oid_nil;
sql_trans *tr;
int ncid; /* dataset id */
int ndims, nvars, ngatts, unlimdim;
int didx, vidx, vndims, vnatts, i, aidx, coord_dim_id = -1;
int vdims[NC_MAX_VAR_DIMS];
size_t dlen, alen;
char dname[NC_MAX_NAME+1], vname[NC_MAX_NAME+1], aname[NC_MAX_NAME +1],
abuf[80], *aval;
char **dims = NULL;
nc_type vtype, atype; /* == int */
int retval, avalint;
float avalfl;
double avaldbl;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
if ( !sch )
return createException(MAL, "netcdf.attach", "Cannot get schema sys\n");
tfiles = mvc_bind_table(m, sch, "netcdf_files");
tdims = mvc_bind_table(m, sch, "netcdf_dims");
tvars = mvc_bind_table(m, sch, "netcdf_vars");
tvardim = mvc_bind_table(m, sch, "netcdf_vardim");
tattrs = mvc_bind_table(m, sch, "netcdf_attrs");
if (tfiles == NULL || tdims == NULL || tvars == NULL ||
tvardim == NULL || tattrs == NULL)
return createException(MAL, "netcdf.attach", "Catalog table missing\n");
/* check if the file is already attached */
col = mvc_bind_column(m, tfiles, "location");
rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL);
if (rid != oid_nil)
return createException(SQL, "netcdf.attach", "File %s is already attached\n", fname);
/* Open NetCDF file */
if ((retval = nc_open(fname, NC_NOWRITE, &ncid)))
return createException(MAL, "netcdf.test", "Cannot open NetCDF \
file %s: %s", fname, nc_strerror(retval));
if ((retval = nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdim)))
return createException(MAL, "netcdf.test", "Cannot read NetCDF \
header: %s", nc_strerror(retval));
/* Insert row into netcdf_files table */
col = mvc_bind_column(m, tfiles, "file_id");
fid = store_funcs.count_col(tr, col, 1) + 1;
snprintf(buf, BUFSIZ, INSFILE, (int)fid, fname);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
/* Read dimensions from NetCDF header and insert a row for each one into netcdf_dims table */
dims = (char **)GDKzalloc(sizeof(char *) * ndims);
for (didx = 0; didx < ndims; didx++){
if ((retval = nc_inq_dim(ncid, didx, dname, &dlen)))
return createException(MAL, "netcdf.attach", "Cannot read dimension %d : %s", didx, nc_strerror(retval));
snprintf(buf, BUFSIZ, INSDIM, didx, (int)fid, dname, (int)dlen);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
dims[didx] = GDKstrdup(dname);
}
/* Read variables and attributes from the header and insert rows in netcdf_vars, netcdf_vardims, and netcdf_attrs tables */
for (vidx = 0; vidx < nvars; vidx++){
if ( (retval = nc_inq_var(ncid, vidx, vname, &vtype, &vndims, vdims, &vnatts)))
return createException(MAL, "netcdf.attach",
"Cannot read variable %d : %s",
vidx, nc_strerror(retval));
/* Check if this is coordinate variable */
if ( (vndims == 1) && ( strcmp(vname, dims[vdims[0]]) == 0 ))
coord_dim_id = vdims[0];
else coord_dim_id = -1;
snprintf(buf, BUFSIZ, INSVAR, vidx, (int)fid, vname, prim_type_name(vtype), vndims, coord_dim_id);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
if ( coord_dim_id < 0 ){
for (i = 0; i < vndims; i++){
snprintf(buf, BUFSIZ, INSVARDIM, vidx, vdims[i], (int)fid, i);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
}
}
if ( vnatts > 0 ) { /* fill in netcdf_attrs table */
for (aidx = 0; aidx < vnatts; aidx++){
if ((retval = nc_inq_attname(ncid,vidx,aidx,aname)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %d of variable %d: %s",
aidx, vidx, nc_strerror(retval));
if ((retval = nc_inq_att(ncid,vidx,aname,&atype,&alen)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s type and length: %s",
aname, nc_strerror(retval));
switch ( atype ) {
case NC_CHAR:
aval = (char *) GDKzalloc(alen + 1);
if ((retval = nc_get_att_text(ncid,vidx,aname,aval)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
fix_quote(aval, alen);
aval[alen] = '\0';
snprintf(buf, BUFSIZ, INSATTR, vname, aname, "string", aval, (int)fid, "root");
GDKfree(aval);
break;
case NC_INT:
if ((retval = nc_get_att_int(ncid,vidx,aname,&avalint)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%d",avalint);
snprintf(buf, BUFSIZ, INSATTR, vname, aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_FLOAT:
if ((retval = nc_get_att_float(ncid,vidx,aname,&avalfl)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%7.2f",avalfl);
snprintf(buf, BUFSIZ, INSATTR, vname, aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_DOUBLE:
if ((retval = nc_get_att_double(ncid,vidx,aname,&avaldbl)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%7.2e",avaldbl);
snprintf(buf, BUFSIZ, INSATTR, vname, aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
default: continue; /* next attribute */
}
printf("statement: '%s'\n", s);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
} /* attr loop */
}
} /* var loop */
/* Extract global attributes */
for (aidx = 0; aidx < ngatts; aidx++){
if ((retval = nc_inq_attname(ncid,NC_GLOBAL,aidx,aname)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %d: %s",
aidx, nc_strerror(retval));
if ((retval = nc_inq_att(ncid,NC_GLOBAL,aname,&atype,&alen)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s type and length: %s",
aname, nc_strerror(retval));
switch ( atype ) {
case NC_CHAR:
aval = (char *) GDKzalloc(alen + 1);
if ((retval = nc_get_att_text(ncid,NC_GLOBAL,aname,aval)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s value: %s",
aname, nc_strerror(retval));
fix_quote(aval, alen);
aval[alen] = '\0';
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, "string", aval, (int)fid, "root");
GDKfree(aval);
break;
case NC_INT:
if ((retval = nc_get_att_int(ncid,NC_GLOBAL,aname,&avalint)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s of type %s : %s",
aname, prim_type_name(atype), nc_strerror(retval));
snprintf(abuf,80,"%d",avalint);
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_FLOAT:
if ((retval = nc_get_att_float(ncid,NC_GLOBAL,aname,&avalfl)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s of type %s: %s",
aname, prim_type_name(atype), nc_strerror(retval));
snprintf(abuf,80,"%7.2f",avalfl);
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_DOUBLE:
if ((retval = nc_get_att_double(ncid,NC_GLOBAL,aname,&avaldbl)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%7.2e",avaldbl);
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
default: continue; /* next attribute */
}
printf("global: '%s'\n", s);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
} /* global attr loop */
finish:
nc_close(ncid);
if (dims != NULL ){
for (didx = 0; didx < ndims; didx++)
GDKfree(dims[didx]);
GDKfree(dims);
}
return msg;
}
str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_trans *tr;
sql_schema *sch;
sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL;
sql_column *col;
str msg = MAL_SUCCEED;
str fname = *(str*)getArgReference(stk, pci, 1);
fitsfile *fptr; /* pointer to the FITS file */
int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0;
oid fid, tid, cid, rid = oid_nil;
char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ], stmt[BUFSIZ];
long tbcol;
char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ];
double tscal, tzero;
char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = "";
char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = "";
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.attach", "Missing FITS file %s.\n", fname);
return msg;
}
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
fits_fl = mvc_bind_table(m, sch, "fits_files");
if (fits_fl == NULL)
FITSinitCatalog(m);
fits_fl = mvc_bind_table(m, sch, "fits_files");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
fits_col = mvc_bind_table(m, sch, "fits_columns");
fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
/* check if the file is already attached */
col = mvc_bind_column(m, fits_fl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL);
if (rid != oid_nil) {
fits_close_file(fptr, &status);
msg = createException(SQL, "fits.attach", "File %s already attached\n", fname);
return msg;
}
/* add row in the fits_files catalog table */
col = mvc_bind_column(m, fits_fl, "id");
fid = store_funcs.count_col(tr, col, 1) + 1;
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_fl, "id"), &fid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_fl, "name"), fname, TYPE_str);
col = mvc_bind_column(m, fits_tbl, "id");
tid = store_funcs.count_col(tr, col, 1) + 1;
if ((s = strrchr(fname, DIR_SEP)) == NULL)
s = fname;
else
s++;
strcpy(bname, s);
s = strrchr(bname, '.');
if (s) *s = 0;
fits_get_num_hdus(fptr, &hdunum, &status);
for (i = 1; i <= hdunum; i++) {
fits_movabs_hdu(fptr, i, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL)
continue;
/* SQL table name - the name of FITS extention */
fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status);
if (status) {
snprintf(tname, BUFSIZ, "%s_%d", bname, i);
tname_low = toLower(tname);
status = 0;
}else { /* check table name for existence in the fits catalog */
tname_low = toLower(tname);
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname_low, NULL);
/* or as regular SQL table */
tbl = mvc_bind_table(m, sch, tname_low);
if (rid != oid_nil || tbl) {
snprintf(tname, BUFSIZ, "%s_%d", bname, i);
tname_low = toLower(tname);
}
}
fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status);
if (status) {
status = 0;
}
fits_get_num_cols(fptr, &cnum, &status);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "id"), &tid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "name"), tname_low, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "columns"), &cnum, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "file_id"), &fid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "hdu"), &i, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "date"), tdate, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "origin"), orig, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "comment"), comm, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "table_id"), &tid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "xtension"), xtensionname, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "bitpix"), &bitpixnumber, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "stilvers"), stilversion, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "stilclas"), stilclass, TYPE_str);
/* read columns description */
s = stmt;
col = mvc_bind_column(m, fits_col, "id");
cid = store_funcs.count_col(tr, col, 1) + 1;
for (j = 1; j <= cnum; j++, cid++) {
fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status);
snprintf(stmt, BUFSIZ, FITS_INS_COL, (int)cid, cname, tform, tunit, j, (int)tid);
msg = SQLstatementIntern(cntxt, &s, "fits.attach", TRUE, FALSE);
if (msg != MAL_SUCCEED) {
fits_close_file(fptr, &status);
return msg;
}
}
tid++;
}
fits_close_file(fptr, &status);
return MAL_SUCCEED;
}
str
MseedImport(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
int *ret = (int*) getArgReference(stk,pci,0);
int *vid = (int*) getArgReference(stk,pci,1);
str *targetfile = (str*) getArgReference(stk,pci,2);
str msg = MAL_SUCCEED;
MSRecord *msr = 0;
int verbose = 1;
//int ppackets = 2;
int reclen = -1;
int dataflag = 1;
int retcode;
int j;
int sampleindex = 0;
time_t t;
struct tm *tm;
char file[BUFSIZ];
char buf[BUFSIZ], *s= buf;
char starttime[BUFSIZ];
int imin = INT_MAX, imax = INT_MIN;
/* keep state of a file to detect major deviances */
str network =0, station = 0 , location = 0 , channel = 0;
char sampletype = 0;
(void) mb;
*ret = int_nil;
snprintf(file,BUFSIZ,"%s%c%s",vaultpath,DIR_SEP,*targetfile);
if ( access(file,R_OK) )
throw(MAL, "mseed.load", "Cannot access %s\n", file);
while ( (retcode = ms_readmsr (&msr, file, reclen, NULL, NULL, 1, dataflag, verbose)) == MS_NOERROR )
{
if ( network == 0){
network= GDKstrdup(msr->network);
station= GDKstrdup(msr->station);
location= GDKstrdup(msr->location);
channel= GDKstrdup(msr->channel);
sampletype = msr->sampletype;
} else {
if( strcmp(network,msr->network))
msg = createException(MAL,"mseed.import","network name is not stable");
if( strcmp(station,msr->station))
msg = createException(MAL,"mseed.import","station name is not stable");
if( strcmp(location,msr->location))
msg = createException(MAL,"mseed.import","location name is not stable");
if( strcmp(channel,msr->channel))
msg = createException(MAL,"mseed.import","channel name is not stable");
if ( sampletype != msr->sampletype)
msg = createException(MAL,"mseed.import","sample type is not stable");
if (msg) goto wrapup;
}
t= MS_HPTIME2EPOCH(msr->starttime);
tm = gmtime(&t);
snprintf(starttime,BUFSIZ,"%d-%02d-%02d %02d:%02d:%02d.%06d", tm->tm_year +(tm->tm_year > 80?1900:2000), tm->tm_mon+1,tm->tm_mday, tm->tm_hour, tm->tm_min,tm->tm_sec, (int) (msr->starttime % HPTMODULUS));
/* collect the statistics */
switch(msr->sampletype){
case 'i':
imin = INT_MAX, imax = INT_MIN;
if (msr->datasamples)
for ( j=0;j< msr->samplecnt; j++){
if ( imin > ((int*) msr->datasamples)[j]) imin = ((int*) msr->datasamples)[j];
if ( imax < ((int*) msr->datasamples)[j]) imax = ((int*) msr->datasamples)[j];
}
snprintf(buf,BUFSIZ,QRYinsertI, *vid, msr->sequence_number,msr->dataquality,msr->network, msr->station, msr->location, msr->channel,
starttime,msr->samprate, sampleindex,(lng)msr->samplecnt,"int",imin,imax);
break;
case 'a': case 'f': case 'd':
default:
msg = createException(MAL,"mseed.import","data type not yet implemented");
goto wrapup;
}
if ( ( msg =SQLstatementIntern(cntxt,&s,"mseed.import",TRUE,FALSE)) != MAL_SUCCEED)
break;
sampleindex += msr->samplecnt;
}
wrapup:
/* Make sure everything is cleaned up */
ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0);
if ( network) GDKfree(network);
if ( station) GDKfree(station);
if ( location) GDKfree(location);
if ( channel) GDKfree(channel);
if ( msg)
return msg;
if ( retcode != MS_ENDOFFILE )
throw(MAL, "mseed.dump", "Cannot read %s: %s\n", *targetfile, ms_errorstr(retcode));
*ret = *vid;
return msg;
}
/* attach a single shp file given its name, fill in shp catalog tables */
str
SHPattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch = NULL;
sql_table *fls = NULL, *shps = NULL, *shps_dbf = NULL;
sql_column *col;
str msg = MAL_SUCCEED;
str fname = *(str*)getArgReference(stk, pci, 1);
/* SHP-level descriptor */
char buf[BUFSIZ], temp_buf[BUFSIZ], *s=buf;
int i=0, shpid = 0;
oid fid, rid = oid_nil;
GDALWConnection shp_conn;
GDALWConnection * shp_conn_ptr = NULL;
GDALWSimpleFieldDef * field_definitions;
GDALWSpatialInfo spatial_info;
char *nameToLowerCase = NULL;
if((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
if(!(sch = mvc_bind_schema(m, "sys")))
return createException(MAL, "shp.attach", SQLSTATE(38000) "Schema sys missing\n");
fls = mvc_bind_table(m, sch, "files");
shps = mvc_bind_table(m, sch, "shapefiles");
shps_dbf = mvc_bind_table(m, sch, "shapefiles_dbf");
if (fls == NULL || shps == NULL || shps_dbf == NULL )
return createException(MAL, "shp.attach", SQLSTATE(38000) "Catalog table missing\n");
if ((shp_conn_ptr = GDALWConnect((char *) fname)) == NULL) {
return createException(MAL, "shp.attach", SQLSTATE(38000) "Missing shape file %s\n", fname);
}
shp_conn = *shp_conn_ptr;
/* check if the file is already attached */
col = mvc_bind_column(m, fls, "path");
rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL);
if (!is_oid_nil(rid)) {
GDALWClose(shp_conn_ptr);
return createException(MAL, "shp.attach", SQLSTATE(38000) "File %s already attached\n", fname);
}
/* add row in the files(id, path) catalog table */
col = mvc_bind_column(m, fls, "id");
fid = store_funcs.count_col(m->session->tr, col, 1) + 1;
snprintf(buf, BUFSIZ, INSFILE, (int)fid, fname);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto finish;
/*if (shp_conn.layer == NULL || shp_conn.source == NULL || shp_conn.handler == NULL || shp_conn.driver == NULL) {
msg = createException(MAL, "shp.attach", SQLSTATE(38000) "lol-1\n");
return msg;
}*/
/* add row in the shapefiles catalog table (e.g. the name of the table that will store tha data of the shapefile) */
spatial_info = GDALWGetSpatialInfo(shp_conn);
col = mvc_bind_column(m, shps, "shapefileid");
shpid = store_funcs.count_col(m->session->tr, col, 1) + 1;
nameToLowerCase = toLower(shp_conn.layername);
snprintf(buf, BUFSIZ, INSSHP, shpid, (int)fid, spatial_info.epsg, nameToLowerCase);
GDKfree(nameToLowerCase);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto finish;
/* add information about the fields of the shape file
* one row for each field with info (shapefile_id, field_name, field_type) */
field_definitions = GDALWGetSimpleFieldDefinitions(shp_conn);
if (field_definitions == NULL) {
GDALWClose(&shp_conn);
return createException(MAL, "shp.attach", SQLSTATE(HY001) MAL_MALLOC_FAIL);
}
for (i=0 ; i<shp_conn.numFieldDefinitions ; i++) {
snprintf(buf, BUFSIZ, INSSHPDBF, shpid, field_definitions[i].fieldName, field_definitions[i].fieldType);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto fin;
}
/* create the table that will store the data of the shape file */
temp_buf[0]='\0';
for (i=0 ; i<shp_conn.numFieldDefinitions ; i++) {
nameToLowerCase = toLower(field_definitions[i].fieldName);
if (strcmp(field_definitions[i].fieldType, "Integer") == 0) {
sprintf(temp_buf + strlen(temp_buf), "\"%s\" INT, ", nameToLowerCase);
} else if (strcmp(field_definitions[i].fieldType, "Real") == 0) {
sprintf(temp_buf + strlen(temp_buf), "\"%s\" FLOAT, ", nameToLowerCase);
#if 0
} else if (strcmp(field_definitions[i].fieldType, "Date") == 0) {
sprintf(temp_buf + strlen(temp_buf), "\"%s\" STRING, ", nameToLowerCase);
#endif
} else
sprintf(temp_buf + strlen(temp_buf), "\"%s\" STRING, ", nameToLowerCase);
GDKfree(nameToLowerCase);
}
sprintf(temp_buf + strlen(temp_buf), "geom GEOMETRY ");
snprintf(buf, BUFSIZ, CRTTBL, shp_conn.layername, temp_buf);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.import",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto fin;
fin:
free(field_definitions);
finish:
/* if (msg != MAL_SUCCEED){
snprintf(buf, BUFSIZ,"ROLLBACK;");
SQLstatementIntern(cntxt,&s,"geotiff.attach",TRUE,FALSE));
}*/
GDALWClose(&shp_conn);
return msg;
}
