str
MseedLoadSQL(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
int *ret = (int*) getArgReference(stk,pci,0);
str *targetfile = (str*) getArgReference(stk,pci,1);
str msg = MAL_SUCCEED;
BAT *btime, *bdata, *table;
(void) cntxt;
(void) mb;
/* XXX: BATs of BATs are no longer allowed and this code hasn't worked
* for quite some time anyway */
table = BATnew(TYPE_str,TYPE_bat,0);
if ( table == NULL)
throw(MAL, "mseed.load", MAL_MALLOC_FAIL);
msg = MseedLoadIntern(&btime, &bdata, *targetfile);
if ( msg == MAL_SUCCEED){
BUNins(table, (ptr)"time", (ptr)&btime->batCacheid, FALSE);
BUNins(table, (ptr)"data", (ptr)&bdata->batCacheid, FALSE);
BBPreleaseref(btime->batCacheid);
BBPreleaseref(bdata->batCacheid);
BBPkeepref(*ret= table->batCacheid);
}
return msg;
}
str
ITRbunNext(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
BATiter bi;
BAT *b;
oid *head;
bat *bid;
ValPtr tail;
(void) cntxt;
(void) mb;
head = (oid *) getArgReference(stk, pci, 0);
tail = getArgReference(stk,pci,1);
bid = (bat *) getArgReference(stk, pci, 2);
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "iterator.nextChunk", INTERNAL_BAT_ACCESS);
}
*head = (BUN)*head + 1;
if (*head >= BUNlast(b)) {
*head = oid_nil;
BBPunfix(b->batCacheid);
return MAL_SUCCEED;
}
bi = bat_iterator(b);
VALinit(tail, b->ttype, BUNtail(bi, *(BUN*) head));
BBPunfix(b->batCacheid);
return MAL_SUCCEED;
}
str
MATpackValues(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i,*ret, type, first = 1;
BAT *bn;
(void) cntxt;
type = getArgType(mb,p,first);
bn = BATnew(TYPE_void, type, p->argc, TRANSIENT);
if( bn == NULL)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
if (ATOMvarsized(type)) {
for(i = first; i < p->argc; i++)
BUNappend(bn, stk->stk[getArg(p,i)].val.sval, TRUE);
} else {
for(i = first; i < p->argc; i++)
BUNappend(bn, getArgReference(stk, p, i), TRUE);
}
BATsettrivprop(bn);
BATderiveProps(bn,FALSE);
ret= (int*) getArgReference(stk,p,0);
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
str
MATnewIterator(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int *ret = (int*) getArgReference(stk,p,0);
(void) cntxt;
(void) mb;
if( p->argc == 1){
*ret = 0;
} else
*ret= *(int*) getArgReference(stk,p,1);
return MAL_SUCCEED;
}
str
FCTshutdown(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str mod = *(str*) getArgReference(stk, pci, 1);
str fcn = *(str*) getArgReference(stk, pci, 2);
Symbol s;
(void) mb;
s = findSymbol(cntxt->nspace, putName(mod,strlen(mod)), putName(fcn, strlen(fcn)));
if (s == NULL)
throw(MAL, "factories.shutdown", RUNTIME_OBJECT_MISSING);
shutdownFactory(cntxt,s->def);
return MAL_SUCCEED;
}
str
BOXdeposit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
ValPtr v;
Box box;
(void) cntxt;
(void) mb;
OpenBox("deposit");
name = *(str *) getArgReference(stk, pci, 2);
v = getArgReference(stk,pci,3);
if (depositBox(box, name, getArgType(mb,pci,2), v))
throw(MAL, "box.deposit", OPERATION_FAILED);
return MAL_SUCCEED;
}
str
BOXiterator(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
Box box;
oid *cursor;
ValPtr v;
(void) cntxt;
(void) mb; /*fool compiler */
OpenBox("iterator");
cursor = (oid *) getArgReference(stk, pci, 0);
v = getArgReference(stk,pci,2);
(void) nextBoxElement(box, cursor, v);
return MAL_SUCCEED;
}
str
BOXtake(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
Box box;
ValPtr v;
(void) cntxt;
OpenBox("take");
name = *(str *) getArgReference(stk, pci, 2);
v = getArgReference(stk,pci,0);
if (takeBox(box, name, v, (int) getArgType(mb, pci, 0)))
throw(MAL, "box.take", OPERATION_FAILED);
(void) mb;
return MAL_SUCCEED;
}
str
MATpackValues(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i, type, first = 1;
bat *ret;
BAT *bn;
(void) cntxt;
type = getArgType(mb,p,first);
bn = BATnew(TYPE_void, type, p->argc, TRANSIENT);
if( bn == NULL)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
if (ATOMextern(type)) {
for(i = first; i < p->argc; i++)
BUNappend(bn, stk->stk[getArg(p,i)].val.pval, TRUE);
} else {
for(i = first; i < p->argc; i++)
BUNappend(bn, getArgReference(stk, p, i), TRUE);
}
BATseqbase(bn, 0);
ret= getArgReference_bat(stk,p,0);
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
str FITSdir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str dir = *(str*)getArgReference(stk, pci, 1);
DIR *dp;
struct dirent *ep;
fitsfile *fptr;
char *s;
int status = 0;
(void)mb;
dp = opendir(dir);
if (dp != NULL) {
char stmt[BUFSIZ];
char fname[BUFSIZ];
s = stmt;
while ((ep = readdir(dp)) != NULL) {
snprintf(fname, BUFSIZ, "%s%s", dir, ep->d_name);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (status == 0) {
snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
msg = SQLstatementIntern(cntxt, &s, "fits.listofdir", TRUE, FALSE);
fits_close_file(fptr, &status);
}
}
(void)closedir(dp);
}else
msg = createException(MAL, "listdir", "Couldn't open the directory");
return msg;
}
str
MATproject(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
bat *res_id = (bat*) getArgReference(stk,pci,0);
bat map_id = *(bat*) getArgReference(stk,pci,1);
BAT *res = NULL, *map;
/* rest of the args are parts, (excluding result and map) */
BAT **bats = GDKzalloc(sizeof(BAT*) * pci->argc - 2);
BUN bcnt = 0;
int i, len = pci->argc-2, sorted = 1;
(void) cntxt; (void) mb; (void) stk;
if( bats == NULL)
throw(SQL, "mat.project",MAL_MALLOC_FAIL);
map = BATdescriptor(map_id);
if (!map)
goto error;
for (i=2; i<pci->argc; i++) {
bat id = *(bat*) getArgReference(stk,pci,i);
bats[i-2] = BATdescriptor(id);
if (!bats[i-2])
goto error;
bcnt += BATcount(bats[i-2]);
if (!bats[i-2]->T->sorted)
sorted = 0;
}
assert(bcnt == BATcount(map));
res = MATproject_(map, bats, len );
if (sorted && res)
BATordered(BATmirror(res));
error:
if (map) BBPunfix(map->batCacheid);
if (bats) {
for (i=0; i<len && bats[i]; i++)
BBPunfix(bats[i]->batCacheid);
GDKfree(bats);
}
if (res) {
BATsettrivprop(res);
BBPkeepref( *res_id = res->batCacheid);
return MAL_SUCCEED;
}
throw(SQL, "mat.project","Cannot access descriptor");
}
str
MseedLoad(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
int *ret0 = (int*) getArgReference(stk,pci,0);
int *ret1 = (int*) getArgReference(stk,pci,1);
str *targetfile = (str*) getArgReference(stk,pci,2);
str msg;
BAT *btime, *bdata;
(void) cntxt;
(void) mb;
msg = MseedLoadIntern(&btime, &bdata, *targetfile);
if ( msg == MAL_SUCCEED){
BBPkeepref(*ret0 = btime->batCacheid);
BBPkeepref(*ret1 = bdata->batCacheid);
}
return msg;
}
str
MAThasMoreElements(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int *ret = (int*) getArgReference(stk,p,0);
int i, idx = *ret;
(void) cntxt;
(void) mb;
for(i=1; i< p->argc; i++)
if( *(int*) getArgReference(stk,p,i) == idx){
i++;
break;
}
if( i < p->argc)
*ret= *(int*) getArgReference(stk,p,i);
else
*ret = 0;
(void) mb;
return MAL_SUCCEED;
}
str
BOXopen(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
(void) cntxt;
(void) mb; /*fool compiler */
name = *(str *) getArgReference(stk, pci, 1);
if (openBox(name) != 0)
return MAL_SUCCEED;
throw(MAL, "box.open", BOX_CLOSED);
}
str
BOXclose(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
(void) cntxt;
(void) mb; /*fool compiler */
name = *(str *) getArgReference(stk, pci, 1);
if (closeBox(name, FALSE) == 0)
return MAL_SUCCEED;
throw(MAL, "box.close", BOX_CLOSED);
}
/*
* The pack is an ordinary multi BAT insert. Oid synchronistion
* between pieces should be ensured by the code generators.
* The pack operation could be quite expensive, because it
* may create a really large BAT.
* The slice over a mat helps to avoid constructing intermediates
* that are subsequently reduced.
* Contrary to most operations, NIL arguments are skipped and
* do not produce RUNTIME_OBJECT_MISSING.
*/
static str
MATpackInternal(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i, *ret = (int*) getArgReference(stk,p,0);
BAT *b, *bn;
BUN cap = 0;
int tt = TYPE_any;
(void) cntxt;
(void) mb;
for (i = 1; i < p->argc; i++) {
int bid = stk->stk[getArg(p,i)].val.ival;
b = BBPquickdesc(abs(bid),FALSE);
if (b && bid < 0)
b = BATmirror(b);
if( b ){
assert(BAThdense(b));
if (tt == TYPE_any){
tt = b->ttype;
}
if (!tt && tt != b->ttype)
tt = b->ttype;
cap += BATcount(b);
}
}
if (tt == TYPE_any){
*ret = 0;
return MAL_SUCCEED;
}
bn = BATnew(TYPE_void, tt, cap, TRANSIENT);
if (bn == NULL)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
for (i = 1; i < p->argc; i++) {
b = BATdescriptor(stk->stk[getArg(p,i)].val.ival);
if( b ){
if (BATcount(bn) == 0)
BATseqbase(bn, b->H->seq);
if (BATcount(bn) == 0)
BATseqbase(BATmirror(bn), b->T->seq);
BATappend(bn,b,FALSE);
BBPunfix(b->batCacheid);
}
}
assert(!bn->H->nil || !bn->H->nonil);
assert(!bn->T->nil || !bn->T->nonil);
BATsettrivprop(bn);
BATderiveProps(bn,FALSE);
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
str
BOXrelease(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
Box box;
(void) cntxt;
(void) mb; /*fool compiler */
OpenBox("release");
name = *(str *) getArgReference(stk, pci, 2);
releaseBox(box, name);
return MAL_SUCCEED;
}
str FITSdirpat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str dir = *(str*)getArgReference(stk, pci, 1);
str pat = *(str*)getArgReference(stk, pci, 2);
fitsfile *fptr;
char *s;
int status = 0;
glob_t globbuf;
char fulldirectory[BUFSIZ];
size_t j = 0;
(void)mb;
globbuf.gl_offs = 0;
snprintf(fulldirectory, BUFSIZ, "%s%s", dir, pat);
glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf);
/* mnstr_printf(GDKout,"#fulldir: %s \nSize: %lu\n",fulldirectory, globbuf.gl_pathc);*/
if (globbuf.gl_pathc == 0)
msg = createException(MAL, "listdir", "Couldn't open the directory or there are no files that match the pattern");
for (j = 0; j < globbuf.gl_pathc; j++) {
char stmt[BUFSIZ];
char fname[BUFSIZ];
s = stmt;
snprintf(fname, BUFSIZ, "%s", globbuf.gl_pathv[j]);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (status == 0) {
snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
msg = SQLstatementIntern(cntxt, &s, "fits.listofdirpat", TRUE, FALSE);
fits_close_file(fptr, &status);
}
}
return msg;
}
str
BOXdiscard(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
Box box;
(void) cntxt;
(void) mb; /*fool compiler */
OpenBox("discard");
name = *(str *) getArgReference(stk, pci, 2);
if (discardBox(box, name))
throw(MAL, "box.discard", OPERATION_FAILED);
return MAL_SUCCEED;
}
/*
* Enable incremental packing. The SQL front-end requires
* fixed oid sequences.
*/
str
MATpackIncrement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int *ret = (int*) getArgReference(stk,p,0);
int pieces;
BAT *b, *bb, *bn;
size_t newsize;
(void) cntxt;
b = BATdescriptor( stk->stk[getArg(p,1)].val.ival);
if ( b == NULL)
throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
if ( getArgType(mb,p,2) == TYPE_int){
/* first step, estimate with some slack */
pieces = stk->stk[getArg(p,2)].val.ival;
bn = BATnew(TYPE_void, b->ttype?b->ttype:TYPE_oid, (BUN)(1.2 * BATcount(b) * pieces), TRANSIENT);
if (bn == NULL)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
/* allocate enough space for the strings */
if ( b->T->vheap && bn->T->vheap ){
newsize = b->T->vheap->size * pieces;
if (HEAPextend(bn->T->vheap, newsize, TRUE) < 0)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
}
BATseqbase(bn, b->H->seq);
BATseqbase(BATmirror(bn), b->T->seq);
BATappend(bn,b,FALSE);
assert(!bn->H->nil || !bn->H->nonil);
assert(!bn->T->nil || !bn->T->nonil);
BBPkeepref(*ret = bn->batCacheid);
BBPreleaseref(b->batCacheid);
} else {
/* remaining steps */
bb = BATdescriptor(stk->stk[getArg(p,2)].val.ival);
if ( bb ){
if (BATcount(b) == 0)
BATseqbase(b, bb->H->seq);
if (BATcount(b) == 0)
BATseqbase(BATmirror(b), bb->T->seq);
BATappend(b,bb,FALSE);
}
assert(!b->H->nil || !b->H->nonil);
assert(!b->T->nil || !b->T->nonil);
BBPkeepref(*ret = b->batCacheid);
if( bb)
BBPreleaseref(bb->batCacheid);
}
return MAL_SUCCEED;
}
str
TRNtrans_commit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i, *bid;
BAT *b;
(void) cntxt;
(void) mb;
for (i = p->retc; i < p->argc; i++) {
bid = (int *) getArgReference(stk, p, i);
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "transaction.commit", RUNTIME_OBJECT_MISSING);
}
BATcommit(b);
BBPunfix(b->batCacheid);
}
return MAL_SUCCEED;
}
static str
MATpack2Internal(MalStkPtr stk, InstrPtr p)
{
int i,*ret;
BAT *b, *bn;
BUN cap=0;
b= BATdescriptor(stk->stk[getArg(p,1)].val.ival);
if( b == NULL)
throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
bn = BATcopy(b, b->htype, b->ttype, TRUE, TRANSIENT);
BBPunfix(b->batCacheid);
if( bn == NULL)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
for(i = 2; i < p->argc; i++){
b= BATdescriptor(stk->stk[getArg(p,i)].val.ival);
if( b == NULL){
BBPreleaseref(bn->batCacheid);
throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
}
cap += BATcount(b);
BBPunfix(b->batCacheid);
}
bn = BATextend(bn, cap);
if( bn == NULL)
throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
for( i = 2; i < p->argc; i++){
b= BATdescriptor(stk->stk[getArg(p,i)].val.ival);
if( b == NULL){
BBPreleaseref(bn->batCacheid);
throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
}
BATappend(bn,b,FALSE);
BBPunfix(b->batCacheid);
}
ret= (int*) getArgReference(stk,p,0);
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
str
MKEYhash(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
wrd *res;
ptr val;
int tpe = getArgType(mb,p,1);
(void) cntxt;
res= getArgReference_wrd(stk,p,0);
val= getArgReference(stk,p,1);
switch (ATOMstorage(tpe)) {
case TYPE_bte:
*res = MKEYHASH_bte(val);
break;
case TYPE_sht:
*res = MKEYHASH_sht(val);
break;
case TYPE_int:
case TYPE_flt:
*res = MKEYHASH_int(val);
break;
case TYPE_lng:
case TYPE_dbl:
*res = MKEYHASH_lng(val);
break;
#ifdef HAVE_HGE
case TYPE_hge:
*res = MKEYHASH_hge(val);
break;
#endif
default:
if (ATOMextern(tpe))
*res = ATOMhash(tpe, *(ptr*)val);
else
*res = ATOMhash(tpe, val);
break;
}
return MAL_SUCCEED;
}
str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch;
sql_table *fits_fl, *fits_tbl, *tbl = NULL;
sql_column *col;
sql_subtype tpe;
fitsfile *fptr;
str tname = *(str*)getArgReference(stk, pci, 1);
str fname;
str msg = MAL_SUCCEED;
oid rid = oid_nil, frid = oid_nil;
int status = 0, cnum = 0, fid, hdu, hdutype, i, j, anynull = 0, mtype;
int *tpcode = NULL;
long *rep = NULL, *wid = NULL, rows;
char keywrd[80], **cname, nm[FLEN_VALUE];
ptr nilptr;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
sch = mvc_bind_schema(m, "sys");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
if (fits_tbl == NULL) {
msg = createException(MAL, "fits.loadtable", "FITS catalog is missing.\n");
return msg;
}
tbl = mvc_bind_table(m, sch, tname);
if (tbl) {
msg = createException(MAL, "fits.loadtable", "Table %s is already created.\n", tname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
if (rid == oid_nil) {
msg = createException(MAL, "fits.loadtable", "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
return msg;
}
/* Open FITS file and move to the table HDU */
col = mvc_bind_column(m, fits_tbl, "file_id");
fid = *(int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_fl = mvc_bind_table(m, sch, "fits_files");
col = mvc_bind_column(m, fits_fl, "id");
frid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
col = mvc_bind_column(m, fits_fl, "name");
fname = (char *)table_funcs.column_find_value(m->session->tr, col, frid);
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.loadtable", "Missing FITS file %s.\n", fname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "hdu");
hdu = *(int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_movabs_hdu(fptr, hdu, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
msg = createException(MAL, "fits.loadtable", "HDU %d is not a table.\n", hdu);
fits_close_file(fptr, &status);
return msg;
}
/* create a SQL table to hold the FITS table */
/* col = mvc_bind_column(m, fits_tbl, "columns");
cnum = *(int*) table_funcs.column_find_value(m->session->tr, col, rid); */
fits_get_num_cols(fptr, &cnum, &status);
tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum);
tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
rep = (long *)GDKzalloc(sizeof(long) * cnum);
wid = (long *)GDKzalloc(sizeof(long) * cnum);
cname = (char **)GDKzalloc(sizeof(char *) * cnum);
for (j = 1; j <= cnum; j++) {
/* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
snprintf(keywrd, 80, "TTYPE%d", j);
fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
if (status) {
snprintf(nm, FLEN_VALUE, "column_%d", j);
status = 0;
}
cname[j - 1] = GDKstrdup(toLower(nm));
fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
/* mnstr_printf(cntxt->fdout,"#%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->sqlname); */
mvc_create_column(m, tbl, cname[j - 1], &tpe);
}
/* data load */
fits_get_num_rows(fptr, &rows, &status);
mnstr_printf(cntxt->fdout,"#Loading %ld rows in table %s\n", rows, tname);
for (j = 1; j <= cnum; j++) {
BAT *tmp = NULL;
int time0 = GDKms();
mtype = fits2mtype(tpcode[j - 1]);
nilptr = ATOMnil(mtype);
col = mvc_bind_column(m, tbl, cname[j - 1]);
tmp = BATnew(TYPE_void, mtype, rows);
if ( tmp == NULL){
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
throw(MAL,"fits.load", MAL_MALLOC_FAIL);
}
BATseqbase(tmp, 0);
if (rows > (long)REMAP_PAGE_MAXSIZE)
BATmmap(tmp, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
if (mtype != TYPE_str) {
fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, nilptr, (void *)BUNtloc(bat_iterator(tmp), BUNfirst(tmp)), &anynull, &status);
BATsetcount(tmp, rows);
tmp->tsorted = 0;
tmp->trevsorted = 0;
} else {
/* char *v = GDKzalloc(wid[j-1]);*/
int bsize = 50;
int tm0, tloadtm = 0, tattachtm = 0;
int batch = bsize, k;
char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
for(i = 0; i < bsize; i++)
v[i] = GDKzalloc(wid[j-1]);
for(i = 0; i < rows; i += batch) {
batch = rows - i < bsize ? rows - i: bsize;
tm0 = GDKms();
fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, nilptr, (void *)v, &anynull, &status);
tloadtm += GDKms() - tm0;
tm0 = GDKms();
for(k = 0; k < batch ; k++)
BUNappend(tmp, v[k], TRUE);
tattachtm += GDKms() - tm0;
}
for(i = 0; i < bsize ; i++)
GDKfree(v[i]);
GDKfree(v);
mnstr_printf(cntxt->fdout,"#String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm);
}
if (status) {
char buf[FLEN_ERRMSG + 1];
fits_read_errmsg(buf);
msg = createException(MAL, "fits.loadtable", "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
break;
}
mnstr_printf(cntxt->fdout,"#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0);
store_funcs.append_col(m->session->tr, col, tmp, TYPE_bat);
mnstr_printf(cntxt->fdout,"#Total %d ms\n", GDKms() - time0);
BBPunfix(tmp->batCacheid);
}
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
fits_close_file(fptr, &status);
return msg;
}
str RAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped) {
sql_func * sqlfun = NULL;
str exprStr = *getArgReference_str(stk, pci, pci->retc + 1);
SEXP x, env, retval;
SEXP varname = R_NilValue;
SEXP varvalue = R_NilValue;
ParseStatus status;
int i = 0;
char argbuf[64];
char *argnames = NULL;
size_t argnameslen;
size_t pos;
char* rcall = NULL;
size_t rcalllen;
int ret_cols = 0; /* int because pci->retc is int, too*/
str *args;
int evalErr;
char *msg = MAL_SUCCEED;
BAT *b;
node * argnode;
int seengrp = FALSE;
rapiClient = cntxt;
if (!RAPIEnabled()) {
throw(MAL, "rapi.eval",
"Embedded R has not been enabled. Start server with --set %s=true",
rapi_enableflag);
}
if (!rapiInitialized) {
throw(MAL, "rapi.eval",
"Embedded R initialization has failed");
}
if (!grouped) {
sql_subfunc *sqlmorefun = (*(sql_subfunc**) getArgReference(stk, pci, pci->retc));
if (sqlmorefun) sqlfun = (*(sql_subfunc**) getArgReference(stk, pci, pci->retc))->func;
} else {
sqlfun = *(sql_func**) getArgReference(stk, pci, pci->retc);
}
args = (str*) GDKzalloc(sizeof(str) * pci->argc);
if (args == NULL) {
throw(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
}
// get the lock even before initialization of the R interpreter, as this can take a second and must be done only once.
MT_lock_set(&rapiLock);
env = PROTECT(eval(lang1(install("new.env")), R_GlobalEnv));
assert(env != NULL);
// first argument after the return contains the pointer to the sql_func structure
// NEW macro temporarily renamed to MNEW to allow including sql_catalog.h
if (sqlfun != NULL && sqlfun->ops->cnt > 0) {
int carg = pci->retc + 2;
argnode = sqlfun->ops->h;
while (argnode) {
char* argname = ((sql_arg*) argnode->data)->name;
args[carg] = GDKstrdup(argname);
carg++;
argnode = argnode->next;
}
}
// the first unknown argument is the group, we don't really care for the rest.
argnameslen = 2;
for (i = pci->retc + 2; i < pci->argc; i++) {
if (args[i] == NULL) {
if (!seengrp && grouped) {
args[i] = GDKstrdup("aggr_group");
seengrp = TRUE;
} else {
snprintf(argbuf, sizeof(argbuf), "arg%i", i - pci->retc - 1);
args[i] = GDKstrdup(argbuf);
}
}
argnameslen += strlen(args[i]) + 2; /* extra for ", " */
}
// install the MAL variables into the R environment
// we can basically map values to int ("INTEGER") or double ("REAL")
for (i = pci->retc + 2; i < pci->argc; i++) {
int bat_type = getBatType(getArgType(mb,pci,i));
// check for BAT or scalar first, keep code left
if (!isaBatType(getArgType(mb,pci,i))) {
b = COLnew(0, getArgType(mb, pci, i), 0, TRANSIENT);
if (b == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
if ( getArgType(mb,pci,i) == TYPE_str) {
if (BUNappend(b, *getArgReference_str(stk, pci, i), false) != GDK_SUCCEED) {
BBPreclaim(b);
b = NULL;
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
} else {
if (BUNappend(b, getArgReference(stk, pci, i), false) != GDK_SUCCEED) {
BBPreclaim(b);
b = NULL;
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
}
} else {
b = BATdescriptor(*getArgReference_bat(stk, pci, i));
if (b == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
}
// check the BAT count, if it is bigger than RAPI_MAX_TUPLES, fail
if (BATcount(b) > RAPI_MAX_TUPLES) {
msg = createException(MAL, "rapi.eval",
"Got "BUNFMT" rows, but can only handle "LLFMT". Sorry.",
BATcount(b), (lng) RAPI_MAX_TUPLES);
BBPunfix(b->batCacheid);
goto wrapup;
}
varname = PROTECT(Rf_install(args[i]));
varvalue = bat_to_sexp(b, bat_type);
if (varvalue == NULL) {
msg = createException(MAL, "rapi.eval", "unknown argument type ");
goto wrapup;
}
BBPunfix(b->batCacheid);
// install vector into R environment
Rf_defineVar(varname, varvalue, env);
UNPROTECT(2);
}
/* we are going to evaluate the user function within an anonymous function call:
* ret <- (function(arg1){return(arg1*2)})(42)
* the user code is put inside the {}, this keeps our environment clean (TM) and gives
* a clear path for return values, namely using the builtin return() function
* this is also compatible with PL/R
*/
pos = 0;
argnames = malloc(argnameslen);
if (argnames == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
argnames[0] = '\0';
for (i = pci->retc + 2; i < pci->argc; i++) {
pos += snprintf(argnames + pos, argnameslen - pos, "%s%s",
args[i], i < pci->argc - 1 ? ", " : "");
}
rcalllen = 2 * pos + strlen(exprStr) + 100;
rcall = malloc(rcalllen);
if (rcall == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
snprintf(rcall, rcalllen,
"ret <- as.data.frame((function(%s){%s})(%s), nm=NA, stringsAsFactors=F)\n",
argnames, exprStr, argnames);
free(argnames);
argnames = NULL;
#ifdef _RAPI_DEBUG_
printf("# R call %s\n",rcall);
#endif
x = R_ParseVector(mkString(rcall), 1, &status, R_NilValue);
if (LENGTH(x) != 1 || status != PARSE_OK) {
msg = createException(MAL, "rapi.eval",
"Error parsing R expression '%s'. ", exprStr);
goto wrapup;
}
retval = R_tryEval(VECTOR_ELT(x, 0), env, &evalErr);
if (evalErr != FALSE) {
char* errormsg = strdup(R_curErrorBuf());
size_t c;
if (errormsg == NULL) {
msg = createException(MAL, "rapi.eval", "Error running R expression.");
goto wrapup;
}
// remove newlines from error message so it fits into a MAPI error (lol)
for (c = 0; c < strlen(errormsg); c++) {
if (errormsg[c] == '\r' || errormsg[c] == '\n') {
errormsg[c] = ' ';
}
}
msg = createException(MAL, "rapi.eval",
"Error running R expression: %s", errormsg);
free(errormsg);
goto wrapup;
}
// ret should be a data frame with exactly as many columns as we need from retc
ret_cols = LENGTH(retval);
if (ret_cols != pci->retc) {
msg = createException(MAL, "rapi.eval",
"Expected result of %d columns, got %d", pci->retc, ret_cols);
goto wrapup;
}
// collect the return values
for (i = 0; i < pci->retc; i++) {
SEXP ret_col = VECTOR_ELT(retval, i);
int bat_type = getBatType(getArgType(mb,pci,i));
if (bat_type == TYPE_any || bat_type == TYPE_void) {
getArgType(mb,pci,i) = bat_type;
msg = createException(MAL, "rapi.eval",
"Unknown return value, possibly projecting with no parameters.");
goto wrapup;
}
// hand over the vector into a BAT
b = sexp_to_bat(ret_col, bat_type);
if (b == NULL) {
msg = createException(MAL, "rapi.eval",
"Failed to convert column %i", i);
goto wrapup;
}
// bat return
if (isaBatType(getArgType(mb,pci,i))) {
*getArgReference_bat(stk, pci, i) = b->batCacheid;
} else { // single value return, only for non-grouped aggregations
BATiter li = bat_iterator(b);
if (VALinit(&stk->stk[pci->argv[i]], bat_type,
BUNtail(li, 0)) == NULL) { // TODO BUNtail here
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
}
msg = MAL_SUCCEED;
}
/* unprotect environment, so it will be eaten by the GC. */
UNPROTECT(1);
wrapup:
MT_lock_unset(&rapiLock);
if (argnames)
free(argnames);
if (rcall)
free(rcall);
for (i = 0; i < pci->argc; i++)
GDKfree(args[i]);
GDKfree(args);
return msg;
}
str
NCDFattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch = NULL;
sql_table *tfiles = NULL, *tdims = NULL, *tvars = NULL, *tvardim = NULL, *tattrs = NULL;
sql_column *col;
str msg = MAL_SUCCEED;
str fname = *(str*)getArgReference(stk, pci, 1);
char buf[BUFSIZ], *s= buf;
oid fid, rid = oid_nil;
sql_trans *tr;
int ncid; /* dataset id */
int ndims, nvars, ngatts, unlimdim;
int didx, vidx, vndims, vnatts, i, aidx, coord_dim_id = -1;
int vdims[NC_MAX_VAR_DIMS];
size_t dlen, alen;
char dname[NC_MAX_NAME+1], vname[NC_MAX_NAME+1], aname[NC_MAX_NAME +1],
abuf[80], *aval;
char **dims = NULL;
nc_type vtype, atype; /* == int */
int retval, avalint;
float avalfl;
double avaldbl;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
if ( !sch )
return createException(MAL, "netcdf.attach", "Cannot get schema sys\n");
tfiles = mvc_bind_table(m, sch, "netcdf_files");
tdims = mvc_bind_table(m, sch, "netcdf_dims");
tvars = mvc_bind_table(m, sch, "netcdf_vars");
tvardim = mvc_bind_table(m, sch, "netcdf_vardim");
tattrs = mvc_bind_table(m, sch, "netcdf_attrs");
if (tfiles == NULL || tdims == NULL || tvars == NULL ||
tvardim == NULL || tattrs == NULL)
return createException(MAL, "netcdf.attach", "Catalog table missing\n");
/* check if the file is already attached */
col = mvc_bind_column(m, tfiles, "location");
rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL);
if (rid != oid_nil)
return createException(SQL, "netcdf.attach", "File %s is already attached\n", fname);
/* Open NetCDF file */
if ((retval = nc_open(fname, NC_NOWRITE, &ncid)))
return createException(MAL, "netcdf.test", "Cannot open NetCDF \
file %s: %s", fname, nc_strerror(retval));
if ((retval = nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdim)))
return createException(MAL, "netcdf.test", "Cannot read NetCDF \
header: %s", nc_strerror(retval));
/* Insert row into netcdf_files table */
col = mvc_bind_column(m, tfiles, "file_id");
fid = store_funcs.count_col(tr, col, 1) + 1;
snprintf(buf, BUFSIZ, INSFILE, (int)fid, fname);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
/* Read dimensions from NetCDF header and insert a row for each one into netcdf_dims table */
dims = (char **)GDKzalloc(sizeof(char *) * ndims);
for (didx = 0; didx < ndims; didx++){
if ((retval = nc_inq_dim(ncid, didx, dname, &dlen)))
return createException(MAL, "netcdf.attach", "Cannot read dimension %d : %s", didx, nc_strerror(retval));
snprintf(buf, BUFSIZ, INSDIM, didx, (int)fid, dname, (int)dlen);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
dims[didx] = GDKstrdup(dname);
}
/* Read variables and attributes from the header and insert rows in netcdf_vars, netcdf_vardims, and netcdf_attrs tables */
for (vidx = 0; vidx < nvars; vidx++){
if ( (retval = nc_inq_var(ncid, vidx, vname, &vtype, &vndims, vdims, &vnatts)))
return createException(MAL, "netcdf.attach",
"Cannot read variable %d : %s",
vidx, nc_strerror(retval));
/* Check if this is coordinate variable */
if ( (vndims == 1) && ( strcmp(vname, dims[vdims[0]]) == 0 ))
coord_dim_id = vdims[0];
else coord_dim_id = -1;
snprintf(buf, BUFSIZ, INSVAR, vidx, (int)fid, vname, prim_type_name(vtype), vndims, coord_dim_id);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
if ( coord_dim_id < 0 ){
for (i = 0; i < vndims; i++){
snprintf(buf, BUFSIZ, INSVARDIM, vidx, vdims[i], (int)fid, i);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
}
}
if ( vnatts > 0 ) { /* fill in netcdf_attrs table */
for (aidx = 0; aidx < vnatts; aidx++){
if ((retval = nc_inq_attname(ncid,vidx,aidx,aname)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %d of variable %d: %s",
aidx, vidx, nc_strerror(retval));
if ((retval = nc_inq_att(ncid,vidx,aname,&atype,&alen)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s type and length: %s",
aname, nc_strerror(retval));
switch ( atype ) {
case NC_CHAR:
aval = (char *) GDKzalloc(alen + 1);
if ((retval = nc_get_att_text(ncid,vidx,aname,aval)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
fix_quote(aval, alen);
aval[alen] = '\0';
snprintf(buf, BUFSIZ, INSATTR, vname, aname, "string", aval, (int)fid, "root");
GDKfree(aval);
break;
case NC_INT:
if ((retval = nc_get_att_int(ncid,vidx,aname,&avalint)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%d",avalint);
snprintf(buf, BUFSIZ, INSATTR, vname, aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_FLOAT:
if ((retval = nc_get_att_float(ncid,vidx,aname,&avalfl)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%7.2f",avalfl);
snprintf(buf, BUFSIZ, INSATTR, vname, aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_DOUBLE:
if ((retval = nc_get_att_double(ncid,vidx,aname,&avaldbl)))
return createException(MAL, "netcdf.attach",
"Cannot read attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%7.2e",avaldbl);
snprintf(buf, BUFSIZ, INSATTR, vname, aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
default: continue; /* next attribute */
}
printf("statement: '%s'\n", s);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
} /* attr loop */
}
} /* var loop */
/* Extract global attributes */
for (aidx = 0; aidx < ngatts; aidx++){
if ((retval = nc_inq_attname(ncid,NC_GLOBAL,aidx,aname)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %d: %s",
aidx, nc_strerror(retval));
if ((retval = nc_inq_att(ncid,NC_GLOBAL,aname,&atype,&alen)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s type and length: %s",
aname, nc_strerror(retval));
switch ( atype ) {
case NC_CHAR:
aval = (char *) GDKzalloc(alen + 1);
if ((retval = nc_get_att_text(ncid,NC_GLOBAL,aname,aval)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s value: %s",
aname, nc_strerror(retval));
fix_quote(aval, alen);
aval[alen] = '\0';
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, "string", aval, (int)fid, "root");
GDKfree(aval);
break;
case NC_INT:
if ((retval = nc_get_att_int(ncid,NC_GLOBAL,aname,&avalint)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s of type %s : %s",
aname, prim_type_name(atype), nc_strerror(retval));
snprintf(abuf,80,"%d",avalint);
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_FLOAT:
if ((retval = nc_get_att_float(ncid,NC_GLOBAL,aname,&avalfl)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s of type %s: %s",
aname, prim_type_name(atype), nc_strerror(retval));
snprintf(abuf,80,"%7.2f",avalfl);
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
case NC_DOUBLE:
if ((retval = nc_get_att_double(ncid,NC_GLOBAL,aname,&avaldbl)))
return createException(MAL, "netcdf.attach",
"Cannot read global attribute %s value: %s",
aname, nc_strerror(retval));
snprintf(abuf,80,"%7.2e",avaldbl);
snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", aname, prim_type_name(atype), abuf, (int)fid, "root");
break;
default: continue; /* next attribute */
}
printf("global: '%s'\n", s);
if ( ( msg = SQLstatementIntern(cntxt, &s, "netcdf.attach", TRUE, FALSE, NULL))
!= MAL_SUCCEED )
goto finish;
} /* global attr loop */
finish:
nc_close(ncid);
if (dims != NULL ){
for (didx = 0; didx < ndims; didx++)
GDKfree(dims[didx]);
GDKfree(dims);
}
return msg;
}
/* import variable given file id and variable name */
str
NCDFimportVariable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch = NULL;
sql_table *tfiles = NULL, *arr_table = NULL;
sql_column *col;
str msg = MAL_SUCCEED, vname = *(str*)getArgReference(stk, pci, 2);
str fname = NULL, dimtype = NULL, aname_sys = NULL;
int fid = *(int*)getArgReference(stk, pci, 1);
int varid, vndims, vnatts, i, j, retval;
char buf[BUFSIZ], *s= buf, aname[256], **dname;
oid rid = oid_nil;
int vdims[NC_MAX_VAR_DIMS];
nc_type vtype;
int ncid; /* dataset id */
size_t dlen;
bat vbatid = 0, *dim_bids;
BAT *vbat = NULL, *dimbat;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
sch = mvc_bind_schema(m, "sys");
if ( !sch )
return createException(MAL, "netcdf.importvar", "Cannot get schema sys\n");
tfiles = mvc_bind_table(m, sch, "netcdf_files");
if (tfiles == NULL)
return createException(MAL, "netcdf.importvar", "Catalog table missing\n");
/* get the name of the attached NetCDF file */
col = mvc_bind_column(m, tfiles, "file_id");
if (col == NULL)
return createException(MAL, "netcdf.importvar", "Could not find \"netcdf_files\".\"file_id\"\n");
rid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
if (rid == oid_nil)
return createException(MAL, "netcdf.importvar", "File %d not in the NetCDF vault\n", fid);
col = mvc_bind_column(m, tfiles, "location");
fname = (str)table_funcs.column_find_value(m->session->tr, col, rid);
/* Open NetCDF file */
if ((retval = nc_open(fname, NC_NOWRITE, &ncid)))
return createException(MAL, "netcdf.importvar", "Cannot open NetCDF file %s: %s",
fname, nc_strerror(retval));
/* Get info for variable vname from NetCDF file */
if ( (retval = nc_inq_varid(ncid, vname, &varid)) )
return createException(MAL, "netcdf.importvar",
"Cannot read variable %s: %s",
vname, nc_strerror(retval));
if ( (retval = nc_inq_var(ncid, varid, vname, &vtype, &vndims, vdims, &vnatts)))
return createException(MAL, "netcdf.importvar",
"Cannot read variable %d : %s",
varid, nc_strerror(retval));
/* compose 'create table' statement in the buffer */
dname = (char **) GDKzalloc( sizeof(char *) * vndims);
for (i = 0; i < vndims; i++)
dname[i] = (char *) GDKzalloc(NC_MAX_NAME + 1);
snprintf(aname, 256, "%s%d", vname, fid);
j = snprintf(buf, BUFSIZ,"create table %s.%s( ", sch->base.name, aname);
for (i = 0; i < vndims; i++){
if ((retval = nc_inq_dim(ncid, vdims[i], dname[i], &dlen)))
return createException(MAL, "netcdf.importvar",
"Cannot read dimension %d : %s",
vdims[i], nc_strerror(retval));
if ( dlen <= (int) GDK_bte_max )
dimtype = "TINYINT";
else if ( dlen <= (int) GDK_sht_max )
dimtype = "SMALLINT";
else
dimtype = "INT";
(void)dlen;
j += snprintf(buf + j, BUFSIZ - j, "%s %s, ", dname[i], dimtype);
}
j += snprintf(buf + j, BUFSIZ - j, "value %s);", NCDF2SQL(vtype));
/* execute 'create table ' */
msg = SQLstatementIntern(cntxt, &s, "netcdf.importvar", TRUE, FALSE, NULL);
if (msg != MAL_SUCCEED)
return msg;
/* load variable data */
dim_bids = (bat *)GDKmalloc(sizeof(bat) * vndims);
msg = NCDFloadVar(&dim_bids, &vbatid, ncid, varid, vtype, vndims, vdims);
if ( msg != MAL_SUCCEED )
return msg;
/* associate columns in the table with loaded variable data */
aname_sys = toLower(aname);
arr_table = mvc_bind_table(m, sch, aname_sys);
if (arr_table == NULL)
return createException(MAL, "netcdf.importvar", "netcdf table %s missing\n", aname_sys);
col = mvc_bind_column(m, arr_table, "value");
if (col == NULL)
return createException(MAL, "netcdf.importvar", "Cannot find column %s.value\n", aname_sys);
vbat = BATdescriptor(vbatid);
store_funcs.append_col(m->session->tr, col, vbat, TYPE_bat);
BBPunfix(vbatid);
BBPdecref(vbatid, 1);
vbat = NULL;
/* associate dimension bats */
for (i = 0; i < vndims; i++){
col = mvc_bind_column(m, arr_table, dname[i]);
if (col == NULL)
return createException(MAL, "netcdf.importvar", "Cannot find column %s.%s\n", aname_sys, dname[i]);
dimbat = BATdescriptor(dim_bids[i]);
store_funcs.append_col(m->session->tr, col, dimbat, TYPE_bat);
BBPunfix(dim_bids[i]); /* phys. ref from BATdescriptor */
BBPdecref(dim_bids[i], 1); /* log. ref. from loadVar */
dimbat = NULL;
}
for (i = 0; i < vndims; i++)
GDKfree(dname[i]);
GDKfree(dname);
GDKfree(dim_bids);
nc_close(ncid);
return msg;
}
static str
MATsort(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int rev)
{
bat *res_id = (bat*) getArgReference(stk,pci,0); /* result sorted */
bat *map_id = (bat*) getArgReference(stk,pci,1); /* result map */
BAT *res = NULL, *map = NULL;
/* rest of the args are sorted parts, (excluding sorted and map) */
BAT **bats = GDKzalloc(sizeof(BAT*) * pci->argc - 2);
BUN pcnt = 0;
int i, len = pci->argc-2;
(void) cntxt; (void) mb; (void) stk;
if( bats == NULL)
throw(SQL, "mat.sortTail",MAL_MALLOC_FAIL);
for (i=2; i<pci->argc; i++) {
bat id = *(bat*) getArgReference(stk,pci,i);
bats[i-2] = BATdescriptor(id);
if (!bats[i-2])
goto error;
pcnt += BATcount(bats[i-2]);
}
if (ATOMstorage(bats[0]->ttype) <= TYPE_void) {
/*error*/
} else if (ATOMvarsized(bats[0]->ttype)) {
res = MATsort_any(&map, bats, len, pcnt, rev);
} else if (ATOMsize(bats[0]->ttype) == sizeof(bte)) {
res = MATsort_bte(&map, bats, len, pcnt, rev);
} else if (ATOMsize(bats[0]->ttype) == sizeof(sht)) {
res = MATsort_sht(&map, bats, len, pcnt, rev);
} else if (ATOMsize(bats[0]->ttype) == sizeof(int)) {
res = MATsort_int(&map, bats, len, pcnt, rev);
} else if (ATOMsize(bats[0]->ttype) == sizeof(lng)) {
res = MATsort_lng(&map, bats, len, pcnt, rev);
#ifdef HAVE_HGE
} else if (ATOMsize(bats[0]->ttype) == sizeof(hge)) {
res = MATsort_hge(&map, bats, len, pcnt, rev);
#endif
} else {
res = MATsort_any(&map, bats, len, pcnt, rev);
}
if (res) {
res->T->nonil = MATnonil(bats, len);
if (rev) {
res->trevsorted = 1;
res->tsorted = res->batCount <= 1;
} else {
res->tsorted = 1;
res->trevsorted = res->batCount <= 1;
}
}
error:
for (i=0; i<len && bats[i]; i++)
BBPunfix(bats[i]->batCacheid);
GDKfree(bats);
if (map && res) {
map->tsorted = 0;
map->trevsorted = 0;
BBPkeepref( *map_id = map->batCacheid);
BBPkeepref( *res_id = res->batCacheid);
return MAL_SUCCEED;
}
if (map) BBPunfix(map->batCacheid);
throw(SQL, "mat.sortTail","Cannot access descriptor");
}
static str
MATpackSliceInternal(MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i, i1 = p->argc, i2 = -1, *ret = (int*) getArgReference(stk,p,0);
BAT *b, *bn;
BUN cap = 0, fst, lst, cnt, c;
int ht = TYPE_any, tt = TYPE_any;
assert(p->argc > 3);
switch getArgType(mb,p,1) {
case TYPE_wrd:
fst = (BUN) *(wrd*) getArgReference(stk,p,1);
break;
case TYPE_lng:
fst = (BUN) *(lng*) getArgReference(stk,p,1);
break;
case TYPE_int:
fst = (BUN) *(int*) getArgReference(stk,p,1);
break;
default:
throw(MAL, "mat.packSlice", "wrong type for lower bound");
}
switch getArgType(mb,p,2) {
case TYPE_wrd: {
wrd l = *(wrd*) getArgReference(stk,p,2);
if (l == wrd_nil)
lst = BUN_MAX; /* no upper bound */
else
lst = (BUN) l;
break;
}
case TYPE_lng: {
lng l = *(lng*) getArgReference(stk,p,2);
if (l == lng_nil)
lst = BUN_MAX; /* no upper bound */
else
lst = (BUN) l;
break;
}
case TYPE_int: {
int l = *(int*) getArgReference(stk,p,2);
if (l == int_nil)
lst = BUN_MAX; /* no upper bound */
else
lst = (BUN) l;
break;
}
default:
throw(MAL, "mat.packSlice", "wrong type for upper bound");
}
if (lst < BUN_MAX)
lst++; /* inclusive -> exclusive upper bound */
if (lst < fst)
lst = fst;
cnt = lst - fst;
for (i = 3; i < p->argc && cap < lst; i++) {
int bid = stk->stk[getArg(p,i)].val.ival;
b = BBPquickdesc(abs(bid),FALSE);
if (b && bid < 0)
b = BATmirror(b);
if (b == NULL)
throw(MAL, "mat.packSlice", RUNTIME_OBJECT_MISSING);
if (ht == TYPE_any){
ht = b->htype;
tt = b->ttype;
}
c = BATcount(b);
if (cap <= fst) {
/* The optimal case is when the requested slice falls completely in one BAT.
* In that case, we can simply return a slice (view) of that BAT.
* (A pitty that we have calculated the other slices as well.)
*/
if (lst <= cap + c) {
b = BATdescriptor(bid);
if( b){
bn = BATslice(b, fst - cap, lst - cap);
BBPunfix(b->batCacheid);
BBPkeepref(*ret = bn->batCacheid);
} else
throw(MAL, "mat.packSlice", RUNTIME_OBJECT_MISSING);
return MAL_SUCCEED;
}
if (fst < cap + c) {
/* fst falls in BAT i1 == i */
i1 = i;
fst -= cap;
lst -= cap;
cap = 0;
}
}
cap += c;
}
/* lst falls in BAT i2 == i-1 */
i2 = i - 1;
if (cap <= fst) /* i.e., (i1 > i2) */
cap = 0;
else
cap -= fst;
cnt = MIN(cnt, cap);
assert(ht== TYPE_void);
bn = BATnew(TYPE_void, tt, cnt, TRANSIENT);
if (bn == NULL)
throw(MAL, "mat.packSlice", MAL_MALLOC_FAIL);
/* must set seqbase or else BATins will not materialize column */
BATseqbase(bn, 0);
if (tt == TYPE_void)
BATseqbase(BATmirror(bn), 0);
for (i = i1; i <= i2; i++) {
b = BATdescriptor(stk->stk[getArg(p,i)].val.ival);
if (b == NULL){
BBPreleaseref(bn->batCacheid);
throw(MAL, "mat.packSlice", RUNTIME_OBJECT_MISSING);
}
c = BATcount(b);
/* use the right oid ranges, don't change the input */
if (i == i1 && fst > 0) {
BAT *bb = b;
b = BATslice(bb, fst, c);
BBPunfix(bb->batCacheid);
} else
if (i == i2 && lst < c) {
BAT *bb = b;
b = BATslice(bb, 0, lst);
BBPunfix(bb->batCacheid);
}
BATins(bn,b,FALSE);
lst -= c;
BBPunfix(b->batCacheid);
}
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
// merging multiple OID lists, optimized for empty bats
// Further improvement should come from multi-bat merging.
str
MATmergepack(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i,j= 0, *ret = (int*) getArgReference(stk,p,0);
int top=0;
oid **o_end, **o_src, *o, *oo, onxt;
BAT *b, *bn, *bm, **bats;
BUN cap = 0;
(void)cntxt;
(void)mb;
bats = (BAT**) GDKzalloc(sizeof(BAT*) * p->argc);
o_end = (oid**) GDKzalloc(sizeof(oid*) * p->argc);
o_src = (oid**) GDKzalloc(sizeof(oid*) * p->argc);
if ( bats ==0 || o_end == 0 || o_src == 0){
if (bats) GDKfree(bats);
if (o_src) GDKfree(o_src);
if (o_end) GDKfree(o_end);
throw(MAL,"mat.mergepack",MAL_MALLOC_FAIL);
}
for (i = 1; i < p->argc; i++) {
int bid = stk->stk[getArg(p,i)].val.ival;
b = BATdescriptor(abs(bid));
if (b ){
cap += BATcount(b);
if ( BATcount(b) ){
// pre-sort the arguments
onxt = *(oid*) Tloc(b,BUNfirst(b));
for( j =top; j > 0 && onxt < *o_src[j-1]; j--){
o_src[j] = o_src[j-1];
o_end[j] = o_end[j-1];
bats[j] = bats[j-1];
}
o_src[j] = (oid*) Tloc(b,BUNfirst(b));
o_end[j] = o_src[j] + BATcount(b);
bats[j] = b;
top++;
}
}
}
bn = BATnew(TYPE_void, TYPE_oid, cap, TRANSIENT);
if (bn == NULL){
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
}
if ( cap == 0){
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), 0);
BBPkeepref(*ret = bn->batCacheid);
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
return MAL_SUCCEED;
}
BATseqbase(bn, bats[0]->hseqbase);
// UNROLL THE MULTI-BAT MERGE
o = (oid*) Tloc(bn,BUNfirst(bn));
while( top){
*o++ = *o_src[0];
o_src[0]++;
if( o_src[0] == o_end[0]){
// remove this one
for(j=0; j< top; j++){
o_src[j]= o_src[j+1];
o_end[j]= o_end[j+1];
bats[j] = bats[j+1];
}
top--;
} else{
// resort priority queue
onxt= *o_src[0];
for( j=1; j< top && onxt > *o_src[j]; j++){
oo = o_src[j]; o_src[j]= o_src[j-1]; o_src[j-1]= oo;
oo = o_end[j]; o_end[j]= o_end[j-1]; o_end[j-1]= oo;
bm = bats[j]; bats[j]=bats[j-1]; bats[j-1] = bm;
}
}
}
for( i=0; i< top; i++)
BBPunfix(bats[i]->batCacheid);
BATsetcount(bn, (BUN) (o - (oid *) Tloc(bn, BUNfirst(bn))));
BATseqbase(bn, 0);
BATsettrivprop(bn);
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
/* properties */
bn->trevsorted = 0;
bn->tsorted = 1;
bn->tkey = 1;
bn->T->nil = 0;
bn->T->nonil = 1;
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
