static BAT *
MATsort_bte( BAT **map, BAT **bats, int len, BUN cnt, int rev )
{
BAT *res;
int i;
bte *resT, **batsT, *in;
bte *mapT;
BUN len1, len2;
bte *map_in = NULL;
res = BATnew(TYPE_void, bats[0]->ttype, cnt, TRANSIENT);
*map = BATnew(TYPE_void, TYPE_bte, cnt, TRANSIENT);
if (res == NULL || *map == NULL) {
BBPreclaim(res);
BBPreclaim(*map);
*map = NULL;
return NULL;
}
BATseqbase(res, 0);
BATseqbase(*map, 0);
resT = (bte*)Tloc(res, 0);
mapT = (bte*)Tloc(*map, 0);
batsT = (bte**)GDKmalloc(sizeof(bte*) * len);
for (i=0; i<len; i++)
batsT[i] = (bte*)Tloc(bats[i], 0);
/* merge */
in = batsT[0];
len1 = BATcount(bats[0]);
map_in = NULL;
/* TODO: change into a tree version */
for (i=1; i<len; i++) {
len2 = BATcount(bats[i]);
if (rev) {
MATsortloop_bte_rev( resT+cnt-len1-len2,
mapT+cnt-len1-len2,
in, map_in, len1,
batsT[i], i, len2);
} else {
MATsortloop_bte_( resT+cnt-len1-len2,
mapT+cnt-len1-len2,
in, map_in, len1,
batsT[i], i, len2);
}
in = resT+cnt-len1-len2;
map_in = mapT+cnt-len1-len2;
len1 += len2;
}
BATsetcount(res, len1);
BATsetcount(*map, len1);
res->hrevsorted = len1 <= 1;
(*map)->hrevsorted = len1 <= 1;
GDKfree(batsT);
return res;
}
str
DCsliceStrict(int *ret, bat *bid, lng *start, lng *end)
{
BAT *b, *bn = NULL;
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "dcoperator.sliceStrict", "Cannot access descriptor");
}
assert(*start >= 0);
assert(*end >= 0);
assert(*start <= (lng) BUN_MAX);
assert(*end < (lng) BUN_MAX);
assert(*start <= *end);
if ((BUN) ((*end - *start) + 1) > BATcount(b)) {
bn = BATnew(b->htype, b->ttype, 0);
BATsetcount(bn, 0);
*ret = bn->batCacheid;
BBPkeepref(*ret);
return MAL_SUCCEED;
}
bn = BATslice(b, (BUN) *start, (BUN) *end + 1);
BBPreleaseref(b->batCacheid);
if (bn != NULL) {
if (!(bn->batDirty & 2))
bn = BATsetaccess(bn, BAT_READ);
*ret = bn->batCacheid;
BBPkeepref(*ret);
return MAL_SUCCEED;
}
throw(MAL, "dcoperator.sliceStrict", "GDKerror");
}
/*
* @-
* The operator below is only working for a very limited
* case. It also re-uses oids, which may become a semantic
* problem quickly.
*/
str DCdeleteUpperSlice(int *ret, int *bid, int *pos)
{
BAT *b;
int *readerT, *writerT;
BUN size, i;
(void) ret;
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "dc.deleteUpperSlice", "Cannot access input BAT");
/* check for a failure */
assert(b != NULL);
/* remove Hashes etc */
if (b->H->hash)
HASHremove(b);
if (b->T->hash)
HASHremove(BATmirror(b));
size = BATcount(b);
writerT = (int *) Tloc(b, BUNfirst(b));
readerT = (int *) Tloc(b, BUNfirst(b)) + *pos;
for (i = *pos; i < size; i++)
*writerT++ = *readerT++;
b->batInserted -= *pos;
BATsetcount(b, (BUN) (writerT - (int *) Tloc(b, BUNfirst(b))));
BBPunfix(*bid);
b->batDirty = TRUE;
return MAL_SUCCEED;
}
static BAT *
MATproject_hge( BAT *map, BAT **bats, int len, int ttpe )
{
BAT *res;
int i;
BUN j, cnt = BATcount(map);
hge *resT, **batsT;
bte *mapT;
res = BATnew(TYPE_void, ttpe, cnt, TRANSIENT);
batsT = (hge**)GDKmalloc(sizeof(hge*) * len);
if (res == NULL || batsT == NULL) {
if (res)
BBPreclaim(res);
if (batsT)
GDKfree(batsT);
return NULL;
}
BATseqbase(res, map->hseqbase);
resT = (hge*)Tloc(res, 0);
mapT = (bte*)Tloc(map, 0);
for (i=0; i<len; i++)
batsT[i] = (hge*)Tloc(bats[i], 0);
for (j=0; j<cnt; j++)
resT[j] = *batsT[mapT[j]]++;
BATsetcount(res, j);
res->hrevsorted = j <= 1;
GDKfree(batsT);
return res;
}
str DCselectInsert(int *ret, int *res, int *bid, lng *low, lng *hgh)
{
BAT *b, *r;
lng *readerH, *writerH;
lng *readerT, *writerT;
BUN size, i;
(void) ret;
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "dc.selectInsert", "Cannot access input BAT");
if ((r = BATdescriptor(*res)) == NULL)
throw(MAL, "dc.selectInsert", "Cannot access result BAT");
size = BATcount(b);
if (size > BATcapacity(r) - BATcount(r)) {
BUN ncap;
BUN grows;
BUN needed = size - (BATcapacity(r) - BATcount(r));
ncap = BATcapacity(r) + needed;
grows = BATgrows(r);
if (ncap > grows)
grows = ncap;
if (BATextend(r, grows) == NULL)
throw(MAL, "dc.selectInsert", "Failed to make room for the new values");
}
/*printf("in dc.selectInsert size is "OIDFMT,size);*/
writerH = (lng *) Hloc(r, BUNfirst(r));
writerT = (lng *) Tloc(r, BUNfirst(r));
readerH = (lng *) Hloc(b, BUNfirst(b));
readerT = (lng *) Tloc(b, BUNfirst(b));
for (i = 0; i < size; i++) {
if (*readerT >= *low && *readerT <= *hgh) {
*writerH = *readerH;
*writerT = *readerT;
writerH++;
writerT++;
}
readerH++;
readerT++;
}
BATsetcount(r, (BUN) (writerT - (lng *) Tloc(r, BUNfirst(r))));
BBPunfix(*bid);
BBPunfix(*res);
return MAL_SUCCEED;
}
static BAT *
MATsort_any( BAT **map, BAT **bats, int len, BUN cnt, int rev )
{
BAT *res = 0, *in;
int i;
bte *mapT;
BUN len1, len2;
bte *map_in = NULL;
*map = BATnew(TYPE_void, TYPE_bte, cnt, TRANSIENT);
if (*map == NULL)
return NULL;
BATseqbase(*map, 0);
mapT = (bte*)Tloc(*map, 0);
/* merge */
/* TODO: change into a tree version */
in = bats[0];
len1 = BATcount(in);
for (i=1; i<len; i++) {
len2 = BATcount(bats[i]);
if (rev) {
res = MATsortloop_rev(
mapT+cnt-len1-len2,
in, map_in, len1,
bats[i], i, len2);
} else {
res = MATsortloop_(
mapT+cnt-len1-len2,
in, map_in, len1,
bats[i], i, len2);
}
if (i != 1)
BBPunfix(in->batCacheid);
if (res == NULL)
return NULL;
in = res;
map_in = mapT+cnt-len1-len2;
len1 += len2;
}
BATsetcount(*map, len1);
(*map)->hrevsorted = len1 <= 1;
return res;
}
str
BKCdensebat(bat *ret, const wrd *size)
{
BAT *bn;
wrd sz = *size;
if (sz < 0)
sz = 0;
if (sz > (wrd) BUN_MAX)
sz = (wrd) BUN_MAX;
bn = BATnew(TYPE_void, TYPE_void, (BUN) sz, TRANSIENT);
if (bn == NULL)
throw(MAL, "bat.densebat", GDK_EXCEPTION);
BATsetcount(bn, (BUN) sz);
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), 0);
*ret = bn->batCacheid;
BBPkeepref(*ret);
return MAL_SUCCEED;
}
/* create and populate a dimension bat */
static str
NCDFARRAYseries(bat *bid, bte start, bte step, int stop, int group, int series)
{
BAT *bn = NULL;
BUN cnt = 0;
cnt = (BUN) ceil(((stop * 1.0 - start) / step)) * group * series ;
if (stop <= (int) GDK_bte_max ) {
bte sta = (bte) start, ste = (bte) step, sto = (bte) stop;
bn = BATnew(TYPE_void, TYPE_bte, cnt, TRANSIENT);
array_series(sta, ste, sto, bte);
} else if (stop <= (int) GDK_sht_max) {
sht sta = (sht) start, ste = (sht) step, sto = (sht) stop;
bn = BATnew(TYPE_void, TYPE_sht, cnt, TRANSIENT);
array_series(sta, ste, sto, sht);
} else {
int sta = (int) start, ste = (int) step, sto = (int) stop;
bn = BATnew(TYPE_void, TYPE_int, cnt, TRANSIENT);
array_series(sta, ste, sto, int);
}
if ( bn == NULL)
throw(MAL, "array.series", MAL_MALLOC_FAIL);
BATsetcount(bn, cnt);
BATseqbase(bn,0);
bn->hdense = TRUE;
BATkey(bn, TRUE);
bn->hsorted = 1;
bn->hrevsorted = (cnt <= 1);
bn->tsorted = (cnt <= 1 || (series == 1 && step > 0));
bn->trevsorted = (cnt <= 1 || (series == 1 && step < 0));
bn->T->nonil = TRUE;
BBPkeepref(*bid= bn->batCacheid);
return MAL_SUCCEED;
}
/*
* The remainder are utilities to manipulate the BAT view and not to
* forget some details in the process. It expects a position range in
* the underlying BAT and compensates for outliers.
*/
void
VIEWbounds(BAT *b, BAT *view, BUN l, BUN h)
{
BUN cnt;
BATiter bi = bat_iterator(b);
if (b == NULL || view == NULL) {
GDKerror("VIEWbounds: bat argument missing");
return;
}
if (h > BATcount(b))
h = BATcount(b);
if (h < l)
h = l;
l += BUNfirst(b);
view->batFirst = view->batDeleted = view->batInserted = 0;
cnt = h - l;
view->H->heap.base = (view->htype) ? BUNhloc(bi, l) : NULL;
view->T->heap.base = (view->ttype) ? BUNtloc(bi, l) : NULL;
view->H->heap.size = headsize(view, cnt);
view->T->heap.size = tailsize(view, cnt);
BATsetcount(view, cnt);
BATsetcapacity(view, cnt);
}
gdk_return
BATmaterializeh(BAT *b)
{
int ht;
BUN cnt;
Heap head;
BUN p, q;
oid h, *x;
bte tshift;
BATcheck(b, "BATmaterialize", GDK_FAIL);
assert(!isVIEW(b));
ht = b->htype;
cnt = BATcapacity(b);
head = b->H->heap;
p = BUNfirst(b);
q = BUNlast(b);
assert(cnt >= q - p);
ALGODEBUG fprintf(stderr, "#BATmaterialize(%d);\n", (int) b->batCacheid);
if (!BAThdense(b) || ht != TYPE_void) {
/* no voids */
return GDK_SUCCEED;
}
ht = TYPE_oid;
/* cleanup possible ACC's */
HASHdestroy(b);
IMPSdestroy(b);
b->H->heap.filename = NULL;
if (HEAPalloc(&b->H->heap, cnt, sizeof(oid)) != GDK_SUCCEED) {
b->H->heap = head;
return GDK_FAIL;
}
/* point of no return */
b->htype = ht;
tshift = b->T->shift;
BATsetdims(b);
if (b->ttype) {
b->T->shift = tshift; /* restore in case it got changed */
b->T->width = 1 << tshift;
}
b->batDirty = TRUE;
b->batDirtydesc = TRUE;
b->H->heap.dirty = TRUE;
/* set the correct dense info */
b->hdense = TRUE;
/* So now generate [h..h+cnt-1] */
h = b->hseqbase;
x = (oid *) b->H->heap.base;
for (; p < q; p++)
*x++ = h++;
cnt = h - b->hseqbase;
BATsetcount(b, cnt);
/* cleanup the old heaps */
HEAPfree(&head, 0);
return GDK_SUCCEED;
}
/* BATsample implements sampling for void headed BATs */
BAT *
BATsample(BAT *b, BUN n)
{
BAT *bn;
BUN cnt, slen;
BUN rescnt;
struct oidtreenode *tree = NULL;
BATcheck(b, "BATsample", NULL);
assert(BAThdense(b));
ERRORcheck(n > BUN_MAX, "BATsample: sample size larger than BUN_MAX\n", NULL);
ALGODEBUG
fprintf(stderr, "#BATsample: sample " BUNFMT " elements.\n", n);
cnt = BATcount(b);
/* empty sample size */
if (n == 0) {
bn = BATnew(TYPE_void, TYPE_void, 0, TRANSIENT);
if (bn == NULL) {
GDKerror("BATsample: memory allocation error");
return NULL;
}
BATsetcount(bn, 0);
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), 0);
/* sample size is larger than the input BAT, return all oids */
} else if (cnt <= n) {
bn = BATnew(TYPE_void, TYPE_void, cnt, TRANSIENT);
if (bn == NULL) {
GDKerror("BATsample: memory allocation error");
return NULL;
}
BATsetcount(bn, cnt);
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), b->H->seq);
} else {
oid minoid = b->hseqbase;
oid maxoid = b->hseqbase + cnt;
/* if someone samples more than half of our tree, we
* do the antiset */
bit antiset = n > cnt / 2;
slen = n;
if (antiset)
n = cnt - n;
tree = GDKmalloc(n * sizeof(struct oidtreenode));
if (tree == NULL) {
GDKerror("#BATsample: memory allocation error");
return NULL;
}
bn = BATnew(TYPE_void, TYPE_oid, slen, TRANSIENT);
if (bn == NULL) {
GDKfree(tree);
GDKerror("#BATsample: memory allocation error");
return NULL;
}
/* while we do not have enough sample OIDs yet */
for (rescnt = 0; rescnt < n; rescnt++) {
oid candoid;
do {
/* generate a new random OID */
candoid = (oid) (minoid + DRAND * (maxoid - minoid));
/* if that candidate OID was already
* generated, try again */
} while (!OIDTreeMaybeInsert(tree, candoid, rescnt));
}
if (!antiset) {
OIDTreeToBAT(tree, bn);
} else {
OIDTreeToBATAntiset(tree, bn, minoid, maxoid);
}
GDKfree(tree);
BATsetcount(bn, slen);
bn->trevsorted = bn->batCount <= 1;
bn->tsorted = 1;
bn->tkey = 1;
bn->tdense = bn->batCount <= 1;
if (bn->batCount == 1)
bn->tseqbase = *(oid *) Tloc(bn, BUNfirst(bn));
bn->hdense = 1;
bn->hseqbase = 0;
bn->hkey = 1;
bn->hrevsorted = bn->batCount <= 1;
bn->hsorted = 1;
}
return bn;
}
str
BKCreuseBATmap(bat *ret, const bat *bid, const bat *did)
{
BAT *b, *d, *bn, *bs;
oid bidx, oidx = 0, *o, *ol;
oid *r;
gdk_return res;
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "bat.shrinkMap", RUNTIME_OBJECT_MISSING);
}
if ( b->htype != TYPE_void) {
BBPunfix(b->batCacheid);
throw(MAL, "bat.shrinkMap", SEMANTIC_TYPE_MISMATCH);
}
if ((d = BATdescriptor(*did)) == NULL) {
BBPunfix(b->batCacheid);
throw(MAL, "bat.shrinkMap", RUNTIME_OBJECT_MISSING);
}
bn= BATnew(TYPE_void, TYPE_oid, BATcount(b) - BATcount(d), TRANSIENT);
if (bn == NULL) {
BBPunfix(b->batCacheid);
BBPunfix(d->batCacheid);
throw(MAL, "bat.shrinkMap", MAL_MALLOC_FAIL );
}
res = BATsubsort(&bs, NULL, NULL, d, NULL, NULL, 0, 0);
BBPunfix(d->batCacheid);
if (res != GDK_SUCCEED) {
BBPunfix(b->batCacheid);
BBPunfix(bn->batCacheid);
throw(MAL, "bat.shrinkMap", MAL_MALLOC_FAIL );
}
oidx = b->hseqbase;
bidx = oidx + BATcount(b)-1;
o = (oid*)Tloc(bs, BUNfirst(bs));
ol = (oid*)Tloc(bs, BUNlast(bs));
r = (oid*)Tloc(bn, BUNfirst(bn));
for (; oidx <= bidx; oidx++) {
if ( *o == oidx ){
while ( ol > o && ol[-1] == bidx) {
bidx--;
ol--;
}
*r++ = bidx;
o += (o < ol);
bidx--;
} else {
*r++ = oidx;
}
}
BATsetcount(bn, BATcount(b)-BATcount(bs));
BATseqbase(bn, b->hseqbase);
bn->tsorted = 0;
bn->trevsorted = 0;
bn->tdense = 0;
if (!(bn->batDirty&2)) BATsetaccess(bn, BAT_READ);
BBPunfix(b->batCacheid);
BBPunfix(bs->batCacheid);
BBPkeepref(*ret= bn->batCacheid);
return MAL_SUCCEED;
}
/* Load variable varid from data set ncid into the bat v. Generate dimension
* bats dim using NCDFARRAYseries */
static str
NCDFloadVar(bat **dim, bat *v, int ncid, int varid, nc_type vtype, int vndims, int *vdims)
{
BAT *res;
bat vbid, *dim_bids;
int retval, i, j;
char *sermsg = NULL;
size_t sz = 1;
size_t *dlen = NULL, *val_rep = NULL, *grp_rep = NULL;
if ( dim == NULL )
return createException(MAL, "netcdf.importvar", "array of dimension bat is NULL");
dim_bids = *dim;
dlen = (size_t *)GDKzalloc(sizeof(size_t) * vndims);
for (i = 0; i < vndims; i++){
if ((retval = nc_inq_dimlen(ncid, vdims[i], &dlen[i])))
return createException(MAL, "netcdf.importvar",
"Cannot read dimension %d : %s",
vdims[i], nc_strerror(retval));
sz *= dlen[i];
}
switch (vtype) {
case NC_INT:
{
LOAD_NCDF_VAR(int,int);
break;
}
case NC_FLOAT:
case NC_DOUBLE:
{
LOAD_NCDF_VAR(dbl,double);
break;
}
default:
return createException(MAL, "netcdf.importvar",
"Type %s not supported yet",
prim_type_name(vtype));
}
BATsetcount(res, sz);
res->T->nonil = TRUE;
res->T->nil = FALSE;
res->tsorted = FALSE;
res->trevsorted = FALSE;
BATkey(BATmirror(res), FALSE);
BBPkeepref(vbid = res->batCacheid);
res = NULL;
/* Manually create dimensions with range [0:1:dlen[i]] */
val_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
grp_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
/* compute the repetition factor inside of the series (val_rep) and of series (grp_rep) */
for (i = 0; i < vndims; i++) {
val_rep[i] = grp_rep[i] = 1;
for (j = 0; j < i; j++)
grp_rep[i] *= dlen[j];
for (j = i + 1; j < vndims; j++)
val_rep[i] *= dlen[j];
}
for (i = 0; i < vndims; i++) {
sermsg = NCDFARRAYseries(&dim_bids[i], 0, 1, dlen[i], val_rep[i], grp_rep[i]);
if (sermsg != MAL_SUCCEED) {
BBPdecref(vbid, 1); /* undo the BBPkeepref(vbid) above */
for ( j = 0; j < i; j++) /* undo log. ref of previous dimensions */
BBPdecref(dim_bids[j], 1);
GDKfree(dlen);
GDKfree(val_rep);
GDKfree(grp_rep);
return createException(MAL, "netcdf.loadvar", "Failed to create a dimension of variable %d", varid);
}
}
/* to do : is descriptor check of dim_bids is needed? */
GDKfree(dlen);
GDKfree(val_rep);
GDKfree(grp_rep);
*v = vbid;
return MAL_SUCCEED;
}
str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch;
sql_table *fits_fl, *fits_tbl, *tbl = NULL;
sql_column *col;
sql_subtype tpe;
fitsfile *fptr;
str tname = *(str*)getArgReference(stk, pci, 1);
str fname;
str msg = MAL_SUCCEED;
oid rid = oid_nil, frid = oid_nil;
int status = 0, cnum = 0, fid, hdu, hdutype, i, j, anynull = 0, mtype;
int *tpcode = NULL;
long *rep = NULL, *wid = NULL, rows;
char keywrd[80], **cname, nm[FLEN_VALUE];
ptr nilptr;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
sch = mvc_bind_schema(m, "sys");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
if (fits_tbl == NULL) {
msg = createException(MAL, "fits.loadtable", "FITS catalog is missing.\n");
return msg;
}
tbl = mvc_bind_table(m, sch, tname);
if (tbl) {
msg = createException(MAL, "fits.loadtable", "Table %s is already created.\n", tname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
if (rid == oid_nil) {
msg = createException(MAL, "fits.loadtable", "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
return msg;
}
/* Open FITS file and move to the table HDU */
col = mvc_bind_column(m, fits_tbl, "file_id");
fid = *(int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_fl = mvc_bind_table(m, sch, "fits_files");
col = mvc_bind_column(m, fits_fl, "id");
frid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
col = mvc_bind_column(m, fits_fl, "name");
fname = (char *)table_funcs.column_find_value(m->session->tr, col, frid);
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.loadtable", "Missing FITS file %s.\n", fname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "hdu");
hdu = *(int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_movabs_hdu(fptr, hdu, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
msg = createException(MAL, "fits.loadtable", "HDU %d is not a table.\n", hdu);
fits_close_file(fptr, &status);
return msg;
}
/* create a SQL table to hold the FITS table */
/* col = mvc_bind_column(m, fits_tbl, "columns");
cnum = *(int*) table_funcs.column_find_value(m->session->tr, col, rid); */
fits_get_num_cols(fptr, &cnum, &status);
tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum);
tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
rep = (long *)GDKzalloc(sizeof(long) * cnum);
wid = (long *)GDKzalloc(sizeof(long) * cnum);
cname = (char **)GDKzalloc(sizeof(char *) * cnum);
for (j = 1; j <= cnum; j++) {
/* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
snprintf(keywrd, 80, "TTYPE%d", j);
fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
if (status) {
snprintf(nm, FLEN_VALUE, "column_%d", j);
status = 0;
}
cname[j - 1] = GDKstrdup(toLower(nm));
fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
/* mnstr_printf(cntxt->fdout,"#%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->sqlname); */
mvc_create_column(m, tbl, cname[j - 1], &tpe);
}
/* data load */
fits_get_num_rows(fptr, &rows, &status);
mnstr_printf(cntxt->fdout,"#Loading %ld rows in table %s\n", rows, tname);
for (j = 1; j <= cnum; j++) {
BAT *tmp = NULL;
int time0 = GDKms();
mtype = fits2mtype(tpcode[j - 1]);
nilptr = ATOMnil(mtype);
col = mvc_bind_column(m, tbl, cname[j - 1]);
tmp = BATnew(TYPE_void, mtype, rows);
if ( tmp == NULL){
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
throw(MAL,"fits.load", MAL_MALLOC_FAIL);
}
BATseqbase(tmp, 0);
if (rows > (long)REMAP_PAGE_MAXSIZE)
BATmmap(tmp, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
if (mtype != TYPE_str) {
fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, nilptr, (void *)BUNtloc(bat_iterator(tmp), BUNfirst(tmp)), &anynull, &status);
BATsetcount(tmp, rows);
tmp->tsorted = 0;
tmp->trevsorted = 0;
} else {
/* char *v = GDKzalloc(wid[j-1]);*/
int bsize = 50;
int tm0, tloadtm = 0, tattachtm = 0;
int batch = bsize, k;
char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
for(i = 0; i < bsize; i++)
v[i] = GDKzalloc(wid[j-1]);
for(i = 0; i < rows; i += batch) {
batch = rows - i < bsize ? rows - i: bsize;
tm0 = GDKms();
fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, nilptr, (void *)v, &anynull, &status);
tloadtm += GDKms() - tm0;
tm0 = GDKms();
for(k = 0; k < batch ; k++)
BUNappend(tmp, v[k], TRUE);
tattachtm += GDKms() - tm0;
}
for(i = 0; i < bsize ; i++)
GDKfree(v[i]);
GDKfree(v);
mnstr_printf(cntxt->fdout,"#String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm);
}
if (status) {
char buf[FLEN_ERRMSG + 1];
fits_read_errmsg(buf);
msg = createException(MAL, "fits.loadtable", "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
break;
}
mnstr_printf(cntxt->fdout,"#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0);
store_funcs.append_col(m->session->tr, col, tmp, TYPE_bat);
mnstr_printf(cntxt->fdout,"#Total %d ms\n", GDKms() - time0);
BBPunfix(tmp->batCacheid);
}
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
fits_close_file(fptr, &status);
return msg;
}
// merging multiple OID lists, optimized for empty bats
// Further improvement should come from multi-bat merging.
str
MATmergepack(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i,j= 0, *ret = (int*) getArgReference(stk,p,0);
int top=0;
oid **o_end, **o_src, *o, *oo, onxt;
BAT *b, *bn, *bm, **bats;
BUN cap = 0;
(void)cntxt;
(void)mb;
bats = (BAT**) GDKzalloc(sizeof(BAT*) * p->argc);
o_end = (oid**) GDKzalloc(sizeof(oid*) * p->argc);
o_src = (oid**) GDKzalloc(sizeof(oid*) * p->argc);
if ( bats ==0 || o_end == 0 || o_src == 0){
if (bats) GDKfree(bats);
if (o_src) GDKfree(o_src);
if (o_end) GDKfree(o_end);
throw(MAL,"mat.mergepack",MAL_MALLOC_FAIL);
}
for (i = 1; i < p->argc; i++) {
int bid = stk->stk[getArg(p,i)].val.ival;
b = BATdescriptor(abs(bid));
if (b ){
cap += BATcount(b);
if ( BATcount(b) ){
// pre-sort the arguments
onxt = *(oid*) Tloc(b,BUNfirst(b));
for( j =top; j > 0 && onxt < *o_src[j-1]; j--){
o_src[j] = o_src[j-1];
o_end[j] = o_end[j-1];
bats[j] = bats[j-1];
}
o_src[j] = (oid*) Tloc(b,BUNfirst(b));
o_end[j] = o_src[j] + BATcount(b);
bats[j] = b;
top++;
}
}
}
bn = BATnew(TYPE_void, TYPE_oid, cap, TRANSIENT);
if (bn == NULL){
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
}
if ( cap == 0){
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), 0);
BBPkeepref(*ret = bn->batCacheid);
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
return MAL_SUCCEED;
}
BATseqbase(bn, bats[0]->hseqbase);
// UNROLL THE MULTI-BAT MERGE
o = (oid*) Tloc(bn,BUNfirst(bn));
while( top){
*o++ = *o_src[0];
o_src[0]++;
if( o_src[0] == o_end[0]){
// remove this one
for(j=0; j< top; j++){
o_src[j]= o_src[j+1];
o_end[j]= o_end[j+1];
bats[j] = bats[j+1];
}
top--;
} else{
// resort priority queue
onxt= *o_src[0];
for( j=1; j< top && onxt > *o_src[j]; j++){
oo = o_src[j]; o_src[j]= o_src[j-1]; o_src[j-1]= oo;
oo = o_end[j]; o_end[j]= o_end[j-1]; o_end[j-1]= oo;
bm = bats[j]; bats[j]=bats[j-1]; bats[j-1] = bm;
}
}
}
for( i=0; i< top; i++)
BBPunfix(bats[i]->batCacheid);
BATsetcount(bn, (BUN) (o - (oid *) Tloc(bn, BUNfirst(bn))));
BATseqbase(bn, 0);
BATsettrivprop(bn);
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
/* properties */
bn->trevsorted = 0;
bn->tsorted = 1;
bn->tkey = 1;
bn->T->nil = 0;
bn->T->nonil = 1;
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
str
MKEYbathash(bat *res, const bat *bid)
{
BAT *b, *dst;
wrd *r;
BUN n;
if ((b = BATdescriptor(*bid)) == NULL)
throw(SQL, "mkey.bathash", RUNTIME_OBJECT_MISSING);
assert(BAThvoid(b) || BAThrestricted(b));
n = BATcount(b);
dst = BATnew(TYPE_void, TYPE_wrd, n, TRANSIENT);
if (dst == NULL) {
BBPunfix(b->batCacheid);
throw(SQL, "mkey.bathash", MAL_MALLOC_FAIL);
}
BATseqbase(dst, b->hseqbase);
BATsetcount(dst, n);
r = (wrd *) Tloc(dst, BUNfirst(dst));
switch (ATOMstorage(b->ttype)) {
case TYPE_void: {
oid o = b->tseqbase;
if (o == oid_nil)
while (n-- > 0)
*r++ = wrd_nil;
else
while (n-- > 0)
*r++ = (wrd) o++;
break;
}
case TYPE_bte: {
bte *v = (bte *) Tloc(b, BUNfirst(b));
while (n-- > 0) {
*r++ = MKEYHASH_bte(v);
v++;
}
break;
}
case TYPE_sht: {
sht *v = (sht *) Tloc(b, BUNfirst(b));
while (n-- > 0) {
*r++ = MKEYHASH_sht(v);
v++;
}
break;
}
case TYPE_int:
case TYPE_flt: {
int *v = (int *) Tloc(b, BUNfirst(b));
while (n-- > 0) {
*r++ = MKEYHASH_int(v);
v++;
}
break;
}
case TYPE_lng:
case TYPE_dbl: {
lng *v = (lng *) Tloc(b, BUNfirst(b));
while (n-- > 0) {
*r++ = MKEYHASH_lng(v);
v++;
}
break;
}
#ifdef HAVE_HGE
case TYPE_hge: {
hge *v = (hge *) Tloc(b, BUNfirst(b));
while (n-- > 0) {
*r++ = MKEYHASH_hge(v);
v++;
}
break;
}
#endif
default: {
BATiter bi = bat_iterator(b);
BUN (*hash)(const void *) = BATatoms[b->ttype].atomHash;
int (*cmp)(const void *, const void *) = ATOMcompare(b->ttype);
void *nil = ATOMnilptr(b->ttype);
BUN i;
const void *v;
BATloop(b, i, n) {
v = BUNtail(bi, i);
if ((*cmp)(v, nil) == 0)
*r++ = wrd_nil;
else
*r++ = (wrd) (*hash)(v);
}
break;
}
}
str
BKCreuseBAT(bat *ret, const bat *bid, const bat *did)
{
BAT *b, *d, *bn, *bs;
oid oidx = 0, bidx, *o, *ol;
gdk_return res;
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "bat.reuse", RUNTIME_OBJECT_MISSING);
}
if ( b->htype != TYPE_void) {
BBPunfix(b->batCacheid);
throw(MAL, "bat.reuse", SEMANTIC_TYPE_MISMATCH);
}
if ((d = BATdescriptor(*did)) == NULL) {
BBPunfix(b->batCacheid);
throw(MAL, "bat.reuse", RUNTIME_OBJECT_MISSING);
}
bn= BATnew(b->htype, b->ttype, BATcount(b) - BATcount(d), TRANSIENT);
if (bn == NULL) {
BBPunfix(b->batCacheid);
BBPunfix(d->batCacheid);
throw(MAL, "bat.reuse", MAL_MALLOC_FAIL );
}
res = BATsubsort(&bs, NULL, NULL, d, NULL, NULL, 0, 0);
BBPunfix(d->batCacheid);
if (res != GDK_SUCCEED) {
BBPunfix(b->batCacheid);
BBPunfix(bn->batCacheid);
throw(MAL, "bat.reuse", MAL_MALLOC_FAIL );
}
oidx = b->hseqbase;
bidx = oidx + BATcount(b)-1;
o = (oid*)Tloc(bs, BUNfirst(bs));
ol= (oid*)Tloc(bs, BUNlast(bs));
switch(ATOMstorage(b->ttype) ){
case TYPE_bte: reuseloop(bte); break;
case TYPE_sht: reuseloop(sht); break;
case TYPE_int: reuseloop(int); break;
case TYPE_lng: reuseloop(lng); break;
#ifdef HAVE_HGE
case TYPE_hge: reuseloop(hge); break;
#endif
case TYPE_flt: reuseloop(flt); break;
case TYPE_dbl: reuseloop(dbl); break;
case TYPE_oid: reuseloop(oid); break;
case TYPE_str: /* to be done based on its index width */
default:
if (ATOMvarsized(bn->ttype)) {
BUN p = BUNfirst(b);
BUN q = BUNlast(b);
BATiter bi = bat_iterator(b);
for (;p<q; oidx++, p++) {
if ( *o == oidx ){
while ( ol > o && ol[-1] == bidx) {
bidx--;
q--;
ol--;
}
BUNappend(bn, BUNtail(bi, --q), FALSE);
o += (o < ol);
bidx--;
} else {
BUNappend(bn, BUNtail(bi, p), FALSE);
}
}
} else {
switch( b->T->width){
case 1:reuseloop(bte); break;
case 2:reuseloop(sht); break;
case 4:reuseloop(int); break;
case 8:reuseloop(lng); break;
#ifdef HAVE_HGE
case 16:reuseloop(hge); break;
#endif
default:
throw(MAL, "bat.shrink", "Illegal argument type");
}
}
}
BATsetcount(bn, BATcount(b) - BATcount(bs));
BATseqbase(bn, b->hseqbase);
bn->tsorted = 0;
bn->trevsorted = 0;
bn->tdense = 0;
bn->tkey = b->tkey;
if (!(bn->batDirty&2)) BATsetaccess(bn, BAT_READ);
BBPunfix(b->batCacheid);
BBPunfix(bs->batCacheid);
BBPkeepref(*ret= bn->batCacheid);
return MAL_SUCCEED;
}
