/*
* Enable incremental packing. The SQL front-end requires
* fixed oid sequences.
*/
str
MATpackIncrement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
bat *ret = getArgReference_bat(stk,p,0);
int pieces;
BAT *b, *bb, *bn;
size_t newsize;
(void) cntxt;
b = BATdescriptor( stk->stk[getArg(p,1)].val.ival);
if ( b == NULL)
throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
if ( getArgType(mb,p,2) == TYPE_int){
/* first step, estimate with some slack */
pieces = stk->stk[getArg(p,2)].val.ival;
bn = BATnew(TYPE_void, b->ttype?b->ttype:TYPE_oid, (BUN)(1.2 * BATcount(b) * pieces), TRANSIENT);
if (bn == NULL)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
/* allocate enough space for the vheap, but not for strings,
* since BATappend does clever things for strings */
if ( b->T->vheap && bn->T->vheap && ATOMstorage(b->ttype) != TYPE_str){
newsize = b->T->vheap->size * pieces;
if (HEAPextend(bn->T->vheap, newsize, TRUE) != GDK_SUCCEED)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
}
BATseqbase(bn, b->H->seq);
BATseqbase(BATmirror(bn), b->T->seq);
BATappend(bn,b,FALSE);
assert(!bn->H->nil || !bn->H->nonil);
assert(!bn->T->nil || !bn->T->nonil);
bn->H->align = (pieces-1);
BBPkeepref(*ret = bn->batCacheid);
BBPunfix(b->batCacheid);
} else {
/* remaining steps */
bb = BATdescriptor(stk->stk[getArg(p,2)].val.ival);
if ( bb ){
if (BATcount(b) == 0)
BATseqbase(b, bb->H->seq);
if (BATcount(b) == 0)
BATseqbase(BATmirror(b), bb->T->seq);
BATappend(b,bb,FALSE);
}
b->H->align--;
if(b->H->align == 0)
BATsetaccess(b, BAT_READ);
assert(!b->H->nil || !b->H->nonil);
assert(!b->T->nil || !b->T->nonil);
BBPkeepref(*ret = b->batCacheid);
if( bb)
BBPunfix(bb->batCacheid);
}
return MAL_SUCCEED;
}
/*
* The pack is an ordinary multi BAT insert. Oid synchronistion
* between pieces should be ensured by the code generators.
* The pack operation could be quite expensive, because it
* may create a really large BAT.
* The slice over a mat helps to avoid constructing intermediates
* that are subsequently reduced.
* Contrary to most operations, NIL arguments are skipped and
* do not produce RUNTIME_OBJECT_MISSING.
*/
static str
MATpackInternal(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i, *ret = (int*) getArgReference(stk,p,0);
BAT *b, *bn;
BUN cap = 0;
int tt = TYPE_any;
(void) cntxt;
(void) mb;
for (i = 1; i < p->argc; i++) {
int bid = stk->stk[getArg(p,i)].val.ival;
b = BBPquickdesc(abs(bid),FALSE);
if (b && bid < 0)
b = BATmirror(b);
if( b ){
assert(BAThdense(b));
if (tt == TYPE_any){
tt = b->ttype;
}
if (!tt && tt != b->ttype)
tt = b->ttype;
cap += BATcount(b);
}
}
if (tt == TYPE_any){
*ret = 0;
return MAL_SUCCEED;
}
bn = BATnew(TYPE_void, tt, cap, TRANSIENT);
if (bn == NULL)
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
for (i = 1; i < p->argc; i++) {
b = BATdescriptor(stk->stk[getArg(p,i)].val.ival);
if( b ){
if (BATcount(bn) == 0)
BATseqbase(bn, b->H->seq);
if (BATcount(bn) == 0)
BATseqbase(BATmirror(bn), b->T->seq);
BATappend(bn,b,FALSE);
BBPunfix(b->batCacheid);
}
}
assert(!bn->H->nil || !bn->H->nonil);
assert(!bn->T->nil || !bn->T->nonil);
BATsettrivprop(bn);
BATderiveProps(bn,FALSE);
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
/*
* @-
* The operator below is only working for a very limited
* case. It also re-uses oids, which may become a semantic
* problem quickly.
*/
str DCdeleteUpperSlice(int *ret, int *bid, int *pos)
{
BAT *b;
int *readerT, *writerT;
BUN size, i;
(void) ret;
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "dc.deleteUpperSlice", "Cannot access input BAT");
/* check for a failure */
assert(b != NULL);
/* remove Hashes etc */
if (b->H->hash)
HASHremove(b);
if (b->T->hash)
HASHremove(BATmirror(b));
size = BATcount(b);
writerT = (int *) Tloc(b, BUNfirst(b));
readerT = (int *) Tloc(b, BUNfirst(b)) + *pos;
for (i = *pos; i < size; i++)
*writerT++ = *readerT++;
b->batInserted -= *pos;
BATsetcount(b, (BUN) (writerT - (int *) Tloc(b, BUNfirst(b))));
BBPunfix(*bid);
b->batDirty = TRUE;
return MAL_SUCCEED;
}
/*
* Destroy a view.
*/
void
VIEWdestroy(BAT *b)
{
assert(isVIEW(b));
/* remove any leftover private hash structures */
if (b->H->hash)
HASHremove(BATmirror(b));
if (b->T->hash)
HASHremove(b);
IMPSdestroy(b);
VIEWunlink(b);
if (b->htype && !b->H->heap.parentid) {
HEAPfree(&b->H->heap, 0);
} else {
b->H->heap.base = NULL;
b->H->heap.filename = NULL;
}
if (b->ttype && !b->T->heap.parentid) {
HEAPfree(&b->T->heap, 0);
} else {
b->T->heap.base = NULL;
b->T->heap.filename = NULL;
}
b->H->vheap = NULL;
b->T->vheap = NULL;
BATfree(b);
}
/*
* the @#VIEWcombine@ routine effortlessly produces a view with double
* vision on the head column.
*/
BAT *
VIEWcombine(BAT *b)
{
BAT *bn = VIEWhcreate(b), *bm;
if (bn == NULL)
return NULL;
bm = BATmirror(bn);
if (bm == NULL)
return NULL;
if (bn->htype != TYPE_void) {
assert(bn->T->vheap == NULL);
bn->T = bn->H;
bm->H = bn->H;
if (bn->T->heap.parentid)
BBPshare(bn->T->heap.parentid);
if (bn->T->vheap) {
assert(bn->T->vheap->parentid != abs(bn->batCacheid));
assert(bn->T->vheap->parentid > 0);
BBPshare(bn->T->vheap->parentid);
}
ALIGNsetH(bn, b);
} else {
BATseqbase(bm, bn->hseqbase);
}
return bn;
}
static str
AGGRgrouped2(bat *retval, bat *bid, bat *eid, int tp,
BAT *(*grpfunc)(BAT *, BAT *, BAT *, BAT *, int, int, int),
int skip_nils,
const char *malfunc)
{
BAT *b, *g, *e;
b = BATdescriptor(*bid); /* [gid,value] */
if (b == NULL)
throw(MAL, "aggr.sum", RUNTIME_OBJECT_MISSING);
g = BATmirror(BATmark(b, 0)); /* [dense,gid] */
e = BATmirror(BATmark(BATmirror(b), 0)); /* [dense,value] */
BBPreleaseref(b->batCacheid);
b = e;
e = BATdescriptor(*eid); /* [gid,any] */
return AGGRgrouped(retval, b, g, e, tp, grpfunc, skip_nils, malfunc);
}
/*
* @-
* The operator below is only working for a very limited cases.
*/
str DCreplaceTailBasedOnHead(int *ret, int *res, int *bid)
{
BAT *b, *r;
oid *readerH_b;
int *writerT_r, *readerT_b;
BUN size_b, size_r, i;
(void) ret;
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "dc.replaceTailBasedOnHead", "Cannot access input BAT");
/* check for a failure */
assert(b != NULL);
if ((r = BATdescriptor(*res)) == NULL)
throw(MAL, "dc.replaceTailBasedOnHead", "Cannot access result BAT");
/* check for a failure */
assert(r != NULL);
/* remove Hashes etc */
if (r->H->hash)
HASHremove(r);
if (r->T->hash)
HASHremove(BATmirror(r));
size_r = BATcount(r);
size_b = BATcount(b);
if ((b->htype == TYPE_void) && (size_b == size_r)) {
writerT_r = (int *) Tloc(r, BUNfirst(r));
readerT_b = (int *) Tloc(b, BUNfirst(b));
for (i = 0; i < size_r; i++) {
*writerT_r = *readerT_b;
writerT_r++;
readerT_b++;
}
} else if ((b->htype != TYPE_void) && (size_b < size_r)) {
readerH_b = (oid *) Hloc(b, BUNfirst(b));
readerT_b = (int *) Tloc(b, BUNfirst(b));
for (i = 0; i < size_b; i++) {
writerT_r = (int *) Tloc(r, BUNfirst(r)) + *readerH_b;
*writerT_r = *readerT_b;
readerH_b++;
readerT_b++;
}
}
BBPunfix(*bid);
BBPunfix(*res);
r->batDirty = TRUE;
return MAL_SUCCEED;
}
char *
GDKgetenv(const char *name)
{
BUN b = BUNfnd(BATmirror(GDKkey), (ptr) name);
if (b != BUN_NONE) {
BATiter GDKenvi = bat_iterator(GDKval);
return BUNtail(GDKenvi, b);
}
return NULL;
}
str
BKCisSortedReverse(bit *res, const bat *bid)
{
BAT *b;
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "bat.isSorted", RUNTIME_OBJECT_MISSING);
}
*res = BATordered_rev(BATmirror(b));
BBPunfix(b->batCacheid);
return MAL_SUCCEED;
}
str
BKCsetkey(bat *res, const bat *bid, const bit *param)
{
BAT *b;
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "bat.setKey", RUNTIME_OBJECT_MISSING);
}
BATkey(BATmirror(b), *param ? BOUND2BTRUE :FALSE);
*res = b->batCacheid;
BBPkeepref(b->batCacheid);
return MAL_SUCCEED;
}
static BAT *
delta_full_bat_( sql_column *c, sql_delta *bat, int temp, BAT *d, BAT *s)
{
/* return full normalized column bat
if (s) {
b := b.semijoin(s);
i := i.semijoin(s);
u := u.semijoin(s);
}
b := b.kunion(i);
b := b.kdiff(u);
b := b.kunion(u);
b := b.kdiff(reverse(d));
*/
BAT *r, *b, *u, *i = temp_descriptor(bat->ibid);
r = i;
if (temp) {
if (s) {
r = BATsemijoin(i,s);
bat_destroy(i);
}
return r;
}
b = temp_descriptor(bat->bid);
u = temp_descriptor(bat->ubid);
if (s) {
BAT *t;
t = BATsemijoin(b,s); bat_destroy(b); b = t;
t = BATsemijoin(i,s); bat_destroy(i); i = t;
t = BATsemijoin(u,s); bat_destroy(u); u = t;
}
assert(b->ttype == i->ttype);
if (BATcount(i)) {
r = BATkunion(b,i); bat_destroy(b); b = r;
}
bat_destroy(i);
if (BATcount(u)) {
r = BATkdiff(b,u); bat_destroy(b); b = r;
assert(b->ttype == u->ttype);
r = BATkunion(b,u); bat_destroy(b); b = r;
}
bat_destroy(u);
if (d && BATcount(d)) {
r = BATkdiff(b,BATmirror(d)); bat_destroy(b); b = r;
}
if (!bat->cached && !c->base.wtime && !s)
bat->cached = temp_descriptor(b->batCacheid);
return b;
}
str
BKCappend_reverse_val_wrap(bat *r, const bat *bid, const void *u)
{
BAT *b;
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "bat.append", RUNTIME_OBJECT_MISSING);
if ((b = setaccess(b, BAT_WRITE)) == NULL)
throw(MAL, "bat.append", OPERATION_FAILED);
if (b->htype >= TYPE_str && ATOMstorage(b->htype) >= TYPE_str) {
if (u == 0 || *(str*)u == 0)
u = (ptr) str_nil;
else
u = (ptr) *(str *)u;
}
b = BATmirror(b);
if (BUNappend(b, u, FALSE) != GDK_SUCCEED) {
BBPunfix(b->batCacheid);
throw(MAL, "bat.append", GDK_EXCEPTION);
}
b = BATmirror(b);
BBPkeepref(*r = b->batCacheid);
return MAL_SUCCEED;
}
str
BKCreverse(bat *ret, const bat *bid)
{
BAT *b, *bn = NULL;
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "bat.reverse", RUNTIME_OBJECT_MISSING);
}
bn = BATmirror(b); /* bn inherits ref from b */
assert(bn != NULL);
*ret = bn->batCacheid;
BBPkeepref(bn->batCacheid);
return MAL_SUCCEED;
}
str
MATproject(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
bat *res_id = (bat*) getArgReference(stk,pci,0);
bat map_id = *(bat*) getArgReference(stk,pci,1);
BAT *res = NULL, *map;
/* rest of the args are parts, (excluding result and map) */
BAT **bats = GDKzalloc(sizeof(BAT*) * pci->argc - 2);
BUN bcnt = 0;
int i, len = pci->argc-2, sorted = 1;
(void) cntxt; (void) mb; (void) stk;
if( bats == NULL)
throw(SQL, "mat.project",MAL_MALLOC_FAIL);
map = BATdescriptor(map_id);
if (!map)
goto error;
for (i=2; i<pci->argc; i++) {
bat id = *(bat*) getArgReference(stk,pci,i);
bats[i-2] = BATdescriptor(id);
if (!bats[i-2])
goto error;
bcnt += BATcount(bats[i-2]);
if (!bats[i-2]->T->sorted)
sorted = 0;
}
assert(bcnt == BATcount(map));
res = MATproject_(map, bats, len );
if (sorted && res)
BATordered(BATmirror(res));
error:
if (map) BBPunfix(map->batCacheid);
if (bats) {
for (i=0; i<len && bats[i]; i++)
BBPunfix(bats[i]->batCacheid);
GDKfree(bats);
}
if (res) {
BATsettrivprop(res);
BBPkeepref( *res_id = res->batCacheid);
return MAL_SUCCEED;
}
throw(SQL, "mat.project","Cannot access descriptor");
}
str
BKCgetKey(bit *ret, const bat *bid)
{
BAT *b;
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "bat.setPersistence", RUNTIME_OBJECT_MISSING);
if (BATcount(b) <= 1) {
*ret = TRUE;
} else {
if (!b->tkey) {
BATderiveHeadProps(BATmirror(b), 1);
}
*ret = b->tkey ? TRUE : FALSE;
}
BBPunfix(b->batCacheid);
return MAL_SUCCEED;
}
str
BKCdensebat(bat *ret, const wrd *size)
{
BAT *bn;
wrd sz = *size;
if (sz < 0)
sz = 0;
if (sz > (wrd) BUN_MAX)
sz = (wrd) BUN_MAX;
bn = BATnew(TYPE_void, TYPE_void, (BUN) sz, TRANSIENT);
if (bn == NULL)
throw(MAL, "bat.densebat", GDK_EXCEPTION);
BATsetcount(bn, (BUN) sz);
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), 0);
*ret = bn->batCacheid;
BBPkeepref(*ret);
return MAL_SUCCEED;
}
static oid
column_find_row(sql_trans *tr, sql_column *c, void *value, ...)
{
va_list va;
BUN q;
BAT *b = NULL, *s = NULL, *r = NULL, *d = NULL;
oid rid = oid_nil;
sql_column *nc;
void *nv;
sql_dbat *bat = c->t->data;
if (bat->dbid)
d = store_funcs.bind_del(tr, c->t, RDONLY);
va_start(va, value);
while ((nc = va_arg(va, sql_column *)) != NULL) {
nv = va_arg(va, void *);
b = full_column(c, d, s);
if (s)
bat_destroy(s);
s = BATselect(b, value, value);
bat_destroy(b);
c = nc;
value = nv;
}
va_end(va);
b = full_column(c, d, s);
if (s)
bat_destroy(s);
if (d)
bat_destroy(d);
r = BATmirror(b);
q = BUNfnd(r, value);
if (q != BUN_NONE) {
BATiter ri = bat_iterator(r);
rid = *(oid *) BUNtail(ri, q);
}
bat_destroy(b);
return rid;
}
/*
* The @#VIEWhead@ routine effortlessly projects out the tail column.
*/
BAT *
VIEWhead(BAT *b)
{
BAT *bn = VIEWhcreate(b), *bm;
BATstore *bs;
if (bn == NULL)
return NULL;
bs = BBP_desc(bn->batCacheid);
bm = BATmirror(bn);
if (bm == NULL)
return NULL;
bm->H = bn->T = &bs->T;
bn->T->type = TYPE_void;
bn->T->varsized = 1;
bn->T->shift = 0;
bn->T->width = 0;
bn->T->heap.parentid = 0;
bn->T->hash = NULL;
bn->T->heap.size = bn->T->heap.free = 0;
bn->T->heap.base = NULL;
BATseqbase(bm, oid_nil);
return bn;
}
/*
* Materialize a view into a normal BAT. If it is a slice, we really
* want to reduce storage of the new BAT.
*/
gdk_return
VIEWreset(BAT *b)
{
bat hp, tp, hvp, tvp;
Heap head, tail, hh, th;
BAT *n = NULL, *v = NULL;
if (b == NULL)
return GDK_FAIL;
hp = VIEWhparent(b);
tp = VIEWtparent(b);
hvp = VIEWvhparent(b);
tvp = VIEWvtparent(b);
if (hp || tp) {
BAT *m;
BATstore *bs;
BUN cnt;
str nme;
size_t nmelen;
/* alloc heaps */
memset(&head, 0, sizeof(Heap));
memset(&tail, 0, sizeof(Heap));
memset(&hh, 0, sizeof(Heap));
memset(&th, 0, sizeof(Heap));
n = BATdescriptor(abs(b->batCacheid)); /* normalized */
if (n == NULL)
goto bailout;
m = BATmirror(n); /* mirror of normalized */
bs = BBP_desc(n->batCacheid);
cnt = BATcount(n) + 1;
nme = BBP_physical(n->batCacheid);
nmelen = nme ? strlen(nme) : 0;
assert(n->batCacheid > 0);
assert(hp || !b->htype);
assert(tp || !b->ttype);
head.farmid = BBPselectfarm(n->batRole, n->htype, offheap);
tail.farmid = BBPselectfarm(n->batRole, n->ttype, offheap);
if (n->htype) {
head.filename = (str) GDKmalloc(nmelen + 12);
if (head.filename == NULL)
goto bailout;
snprintf(head.filename, nmelen + 12, "%s.head", nme);
if (n->htype && HEAPalloc(&head, cnt, Hsize(n)) != GDK_SUCCEED)
goto bailout;
}
if (n->ttype) {
tail.filename = (str) GDKmalloc(nmelen + 12);
if (tail.filename == NULL)
goto bailout;
snprintf(tail.filename, nmelen + 12, "%s.tail", nme);
if (n->ttype && HEAPalloc(&tail, cnt, Tsize(n)) != GDK_SUCCEED)
goto bailout;
}
if (n->H->vheap) {
hh.farmid = BBPselectfarm(n->batRole, n->htype, varheap);
hh.filename = (str) GDKmalloc(nmelen + 12);
if (hh.filename == NULL)
goto bailout;
snprintf(hh.filename, nmelen + 12, "%s.hheap", nme);
if (ATOMheap(n->htype, &hh, cnt) != GDK_SUCCEED)
goto bailout;
}
if (n->T->vheap) {
th.farmid = BBPselectfarm(n->batRole, n->ttype, varheap);
th.filename = (str) GDKmalloc(nmelen + 12);
if (th.filename == NULL)
goto bailout;
snprintf(th.filename, nmelen + 12, "%s.theap", nme);
if (ATOMheap(n->ttype, &th, cnt) != GDK_SUCCEED)
goto bailout;
}
v = VIEWcreate(n, n);
if (v == NULL)
goto bailout;
/* cut the link to your parents */
VIEWunlink(n);
if (hp) {
BBPunshare(hp);
BBPunfix(hp);
}
if (tp) {
BBPunshare(tp);
BBPunfix(tp);
}
if (hvp) {
BBPunshare(hvp);
BBPunfix(hvp);
}
if (tvp) {
BBPunshare(tvp);
BBPunfix(tvp);
}
/* make sure everything points there */
m->S = n->S = &bs->S;
m->T = n->H = &bs->H;
m->H = n->T = &bs->T;
n->H->type = v->H->type;
n->H->varsized = v->H->varsized;
n->H->shift = v->H->shift;
n->H->width = v->H->width;
n->H->seq = v->H->seq;
n->T->type = v->T->type;
n->T->varsized = v->T->varsized;
n->T->shift = v->T->shift;
n->T->width = v->T->width;
n->T->seq = v->T->seq;
n->H->heap.parentid = n->T->heap.parentid = 0;
n->batRestricted = BAT_WRITE;
/* reset BOUND2KEY */
n->H->key = BAThkey(v);
n->T->key = BATtkey(v);
/* copy the heaps */
n->H->heap = head;
n->T->heap = tail;
/* unshare from parents heap */
if (hh.base) {
assert(n->H->vheap == NULL);
n->H->vheap = (Heap *) GDKzalloc(sizeof(Heap));
if (n->H->vheap == NULL)
goto bailout;
*n->H->vheap = hh;
n->H->vheap->parentid = n->batCacheid;
}
if (th.base) {
assert(n->T->vheap == NULL);
n->T->vheap = (Heap *) GDKzalloc(sizeof(Heap));
if (n->T->vheap == NULL)
goto bailout;
*n->T->vheap = th;
n->T->vheap->parentid = n->batCacheid;
}
n->batSharecnt = 0;
n->batCopiedtodisk = 0;
n->batDirty = 1;
/* reset BOUND2KEY */
n->hkey = BAThkey(v);
n->tkey = BATtkey(v);
/* make the BAT empty and insert all again */
DELTAinit(n);
/* reset capacity */
n->batCapacity = cnt;
/* swap n and v in case the original input was reversed, because
* BATins demands (v)oid-headed input */
if (b->batCacheid < 0) {
n = m;
m = BATmirror(v);
} else {
m = v;
}
/* insert all of v in n, and quit */
BATins(n, m, FALSE);
BBPreclaim(v);
BBPunfix(n->batCacheid);
}
return GDK_SUCCEED;
bailout:
BBPreclaim(v);
if (n != NULL)
BBPunfix(n->batCacheid);
HEAPfree(&head, 0);
HEAPfree(&tail, 0);
HEAPfree(&hh, 0);
HEAPfree(&th, 0);
return GDK_FAIL;
}
/* only materialize the tail */
gdk_return
BATmaterializet(BAT *b)
{
return BATmaterializeh(BATmirror(b));
}
void
QOTupdateStatistics(str nme, int actions, lng val)
{
BATiter bi;
BUN p;
oid idx;
int ival=0, *ip= &ival;
lng lval=0, *lp= &lval;
QOTstatisticsInit();
MT_lock_set(&qotlock, "QOT statistics");
p = BUNfnd(BATmirror(qotStat[QOTnames]),(ptr)nme);
if (p == BUN_NONE) {
BUNappend(qotStat[QOTnames], nme, FALSE);
BUNappend(qotStat[QOTcalls], &ival, FALSE);
BUNappend(qotStat[QOTactions], &ival, FALSE);
BUNappend(qotStat[QOTtimings], &lval, FALSE);
p = BUNfnd(BATmirror(qotStat[QOTnames]),(ptr)nme);
if (p == BUN_NONE){
MT_lock_unset(&qotlock, "QOT statistics");
return;
}
}
bi = bat_iterator(qotStat[QOTnames]);
idx = *(oid*) BUNhead(bi,p);
p = BUNfnd(qotStat[QOTcalls],&idx);
if (p == BUN_NONE) {
#ifdef _Q_STATISTICS_DEBUG
mnstr_printf(GDKout,"#Could not access 'calls'\n");
#endif
MT_lock_unset(&qotlock, "QOT statistics");
return;
}
bi = bat_iterator(qotStat[QOTcalls]);
ip = (int*) BUNtail(bi,p);
*ip = *ip+1;
bi.b->tsorted = bi.b->trevsorted = 0;
bi.b->tkey = 0;
p = BUNfnd(qotStat[QOTactions],&idx);
if (p == BUN_NONE){
#ifdef _Q_STATISTICS_DEBUG
mnstr_printf(GDKout,"#Could not access 'actions'\n");
#endif
MT_lock_unset(&qotlock, "QOT statistics");
return;
}
bi = bat_iterator(qotStat[QOTactions]);
ip = (int*) BUNtail(bi,p);
*ip = *ip+ actions;
bi.b->tsorted = bi.b->trevsorted = 0;
bi.b->tkey = 0;
p = BUNfnd(qotStat[QOTtimings],&idx);
if (p == BUN_NONE){
#ifdef _Q_STATISTICS_DEBUG
mnstr_printf(GDKout, "#Could not access 'timings'\n");
#endif
MT_lock_unset(&qotlock, "QOT statistics");
return ;
}
bi = bat_iterator(qotStat[QOTtimings]);
lp = (lng*) BUNtail(bi,p);
*lp = *lp+ val;
bi.b->tsorted = bi.b->trevsorted = 0;
bi.b->tkey = 0;
MT_lock_unset(&qotlock, "QOT statistics");
}
BAT *
VIEWcreate_(BAT *h, BAT *t, int slice_view)
{
BATstore *bs;
BAT *bn;
bat hp = 0, tp = 0, vc = 0;
BATcheck(h, "VIEWcreate_", NULL);
BATcheck(t, "VIEWcreate_", NULL);
if (BATcount(h) != BATcount(t))
slice_view = 1;
bs = BATcreatedesc(h->htype, t->ttype, FALSE, TRANSIENT);
if (bs == NULL)
return NULL;
bn = &bs->B;
hp = VIEWhparent(h);
tp = VIEWtparent(t);
if ((hp == 0 && h->htype != TYPE_void) || h->H->heap.copied)
hp = h->batCacheid;
if ((tp == 0 && t->ttype != TYPE_void) || t->T->heap.copied)
tp = -t->batCacheid;
assert(h->htype != TYPE_void || !hp);
assert(t->ttype != TYPE_void || !tp);
/* the H and T column descriptors are fully copied. We need
* copies because in case of a mark, we are going to override
* a column with a void. Take care to zero the accelerator
* data, though. */
*bn->H = *h->H;
bn->batDeleted = h->batDeleted;
bn->batFirst = h->batFirst;
bn->batInserted = h->batInserted;
bn->batCount = h->batCount;
bn->batCapacity = h->batCapacity;
if (bn->batFirst > 0) {
bn->H->heap.base += h->batFirst * h->H->width;
bn->batFirst = 0;
}
if (h->H == t->T) {
vc = 1;
tp = hp;
bn->T = bn->H;
} else {
*bn->T = *t->T;
if (bn->batCapacity > t->batCapacity)
bn->batCapacity = t->batCapacity;
if (t->batFirst > 0)
bn->T->heap.base += t->batFirst * t->T->width;
if (bn->batCount < t->batCount) {
/* we can't be sure anymore there are nils */
bn->T->nil = 0;
}
}
if (hp)
BBPshare(hp);
if (tp)
BBPshare(tp);
if (bn->H->vheap) {
assert(h->H->vheap);
assert(bn->H->vheap->parentid > 0);
bn->H->vheap->farmid = h->H->vheap->farmid;
BBPshare(bn->H->vheap->parentid);
}
if (bn->T->vheap) {
assert(t->T->vheap);
assert(bn->T->vheap->parentid > 0);
bn->T->vheap->farmid = t->T->vheap->farmid;
BBPshare(bn->T->vheap->parentid);
}
/* note: H/T->heap's points into bs which was just overwritten
* with a copy from the parent(s). Clear the copied flag since
* our heap was not copied from our parent(s) even if our
* parent's heap was copied from its parent(s). */
bn->H->heap.copied = bn->T->heap.copied = 0;
bn->H->props = bn->T->props = NULL;
/* correct values after copy of head and tail info */
if (hp)
bn->H->heap.parentid = hp;
if (tp)
bn->T->heap.parentid = tp;
BATinit_idents(bn);
/* Some bits must be copied individually. */
bn->batDirty = BATdirty(h);
bn->batRestricted = BAT_READ;
if (slice_view || !hp || isVIEW(h))
/* slices are unequal to their parents; cannot use accs */
bn->H->hash = NULL;
else
/* equal pointers to parent mean view uses acc of parent */
bn->H->hash = h->H->hash;
if (slice_view || !tp || isVIEW(t))
bn->T->hash = NULL;
else
bn->T->hash = t->T->hash;
/* imprints are shared, but the check is dynamic */
bn->H->imprints = NULL;
bn->T->imprints = NULL;
BBPcacheit(bs, 1); /* enter in BBP */
/* View of VIEW combine, ie we need to fix the head of the mirror */
if (vc) {
BAT *bm = BATmirror(bn);
bm->H = bn->H;
}
return bn;
}
/* BATsample implements sampling for void headed BATs */
BAT *
BATsample(BAT *b, BUN n)
{
BAT *bn;
BUN cnt, slen;
BUN rescnt;
struct oidtreenode *tree = NULL;
BATcheck(b, "BATsample", NULL);
assert(BAThdense(b));
ERRORcheck(n > BUN_MAX, "BATsample: sample size larger than BUN_MAX\n", NULL);
ALGODEBUG
fprintf(stderr, "#BATsample: sample " BUNFMT " elements.\n", n);
cnt = BATcount(b);
/* empty sample size */
if (n == 0) {
bn = BATnew(TYPE_void, TYPE_void, 0, TRANSIENT);
if (bn == NULL) {
GDKerror("BATsample: memory allocation error");
return NULL;
}
BATsetcount(bn, 0);
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), 0);
/* sample size is larger than the input BAT, return all oids */
} else if (cnt <= n) {
bn = BATnew(TYPE_void, TYPE_void, cnt, TRANSIENT);
if (bn == NULL) {
GDKerror("BATsample: memory allocation error");
return NULL;
}
BATsetcount(bn, cnt);
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), b->H->seq);
} else {
oid minoid = b->hseqbase;
oid maxoid = b->hseqbase + cnt;
/* if someone samples more than half of our tree, we
* do the antiset */
bit antiset = n > cnt / 2;
slen = n;
if (antiset)
n = cnt - n;
tree = GDKmalloc(n * sizeof(struct oidtreenode));
if (tree == NULL) {
GDKerror("#BATsample: memory allocation error");
return NULL;
}
bn = BATnew(TYPE_void, TYPE_oid, slen, TRANSIENT);
if (bn == NULL) {
GDKfree(tree);
GDKerror("#BATsample: memory allocation error");
return NULL;
}
/* while we do not have enough sample OIDs yet */
for (rescnt = 0; rescnt < n; rescnt++) {
oid candoid;
do {
/* generate a new random OID */
candoid = (oid) (minoid + DRAND * (maxoid - minoid));
/* if that candidate OID was already
* generated, try again */
} while (!OIDTreeMaybeInsert(tree, candoid, rescnt));
}
if (!antiset) {
OIDTreeToBAT(tree, bn);
} else {
OIDTreeToBATAntiset(tree, bn, minoid, maxoid);
}
GDKfree(tree);
BATsetcount(bn, slen);
bn->trevsorted = bn->batCount <= 1;
bn->tsorted = 1;
bn->tkey = 1;
bn->tdense = bn->batCount <= 1;
if (bn->batCount == 1)
bn->tseqbase = *(oid *) Tloc(bn, BUNfirst(bn));
bn->hdense = 1;
bn->hseqbase = 0;
bn->hkey = 1;
bn->hrevsorted = bn->batCount <= 1;
bn->hsorted = 1;
}
return bn;
}
/*
* grouped aggregates
*/
static str
AGGRgrouped(bat *retval, BAT *b, BAT *g, BAT *e, int tp,
BAT *(*grpfunc)(BAT *, BAT *, BAT *, BAT *, int, int, int),
int skip_nils,
const char *malfunc)
{
BAT *bn, *t, *map;
if (b == NULL || g == NULL || e == NULL) {
if (b)
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
}
if (tp == TYPE_any && grpfunc == BATgroupmedian)
tp = b->ttype;
if (!BAThdense(b) || !BAThdense(g)) {
/* if b or g don't have a dense head, replace the head with a
* dense sequence */
t = BATjoin(BATmirror(b), g, MIN(BATcount(b), BATcount(g)));
BBPreleaseref(b->batCacheid);
BBPreleaseref(g->batCacheid);
b = BATmirror(BATmark(t, 0));
g = BATmirror(BATmark(BATmirror(t), 0));
BBPreleaseref(t->batCacheid);
}
if (b->hseqbase != g->hseqbase || BATcount(b) != BATcount(g)) {
/* b and g are not aligned: align them by creating a view on
* one or the other */
oid min; /* lowest common oid */
oid max; /* highest common oid */
min = b->hseqbase;
if (min < g->hseqbase)
min = g->hseqbase;
max = b->hseqbase + BATcount(b);
if (g->hseqbase + BATcount(g) < max)
max = g->hseqbase + BATcount(g);
if (b->hseqbase != min || b->hseqbase + BATcount(b) != max) {
if (min >= max)
min = max = b->hseqbase;
t = BATslice(b, BUNfirst(b) + (BUN) (min - b->hseqbase),
BUNfirst(b) + (BUN) (max - b->hseqbase));
BBPreleaseref(b->batCacheid);
b = t;
}
if (g->hseqbase != min || g->hseqbase + BATcount(g) != max) {
if (min >= max)
min = max = g->hseqbase;
t = BATslice(g, BUNfirst(g) + (BUN) (min - g->hseqbase),
BUNfirst(g) + (BUN) (max - g->hseqbase));
BBPreleaseref(g->batCacheid);
g = t;
}
}
if (!BAThdense(e)) {
/* if e doesn't have a dense head, renumber the group ids with
* a dense sequence at the cost of some left joins */
map = BATmark(e, 0); /* [gid,newgid(dense)] */
BBPreleaseref(e->batCacheid);
e = BATmirror(map); /* [newgid(dense),gid] */
t = BATleftjoin(g, map, BATcount(g)); /* [oid,newgid] */
BBPreleaseref(g->batCacheid);
g = t;
} else {
map = NULL;
}
bn = (*grpfunc)(b, g, e, NULL, tp, skip_nils, 1);
if (bn != NULL && (grpfunc == BATgroupmin || grpfunc == BATgroupmax)) {
BAT *bnn = BATouterjoin(bn, b, BATcount(bn));
BBPreleaseref(bn->batCacheid);
bn = bnn;
}
BBPreleaseref(b->batCacheid);
BBPreleaseref(g->batCacheid);
if (map == NULL) /* if map!=NULL, e is mirror of map */
BBPreleaseref(e->batCacheid);
if (bn == NULL) {
char *errbuf = GDKerrbuf;
char *s;
if (map)
BBPreleaseref(map->batCacheid);
if (errbuf && *errbuf) {
if (strncmp(errbuf, "!ERROR: ", 8) == 0)
errbuf += 8;
if (strchr(errbuf, '!') == errbuf + 5) {
s = createException(MAL, malfunc, "%s", errbuf);
} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
s = createException(MAL, malfunc, "%s", s + 2);
} else {
s = createException(MAL, malfunc, "%s", errbuf);
}
*GDKerrbuf = 0;
return s;
}
throw(MAL, malfunc, OPERATION_FAILED);
}
if (map) {
t = BATleftjoin(map, bn, BATcount(bn));
BBPreleaseref(map->batCacheid);
BBPreleaseref(bn->batCacheid);
bn = t;
}
*retval = bn->batCacheid;
BBPkeepref(bn->batCacheid);
return MAL_SUCCEED;
}
static str
AGGRsubgrouped(bat *retval, bat *bid, bat *gid, bat *eid, bat *sid,
int skip_nils, int abort_on_error, int tp,
BAT *(*grpfunc)(BAT *, BAT *, BAT *, BAT *, int, int, int),
const char *malfunc)
{
BAT *b, *g, *e, *s, *bn;
b = BATdescriptor(*bid);
g = gid ? BATdescriptor(*gid) : NULL;
e = eid ? BATdescriptor(*eid) : NULL;
if (b == NULL || (gid != NULL && g == NULL) || (eid != NULL && e == NULL)) {
if (b)
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
}
if (tp == TYPE_any && grpfunc == BATgroupmedian)
tp = b->ttype;
if (sid) {
s = BATdescriptor(*sid);
if (s == NULL) {
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
}
} else {
if (!BAThdense(b)) {
/* XXX backward compatibility code: ignore non-dense head, but
* only if no candidate list */
s = BATmirror(BATmark(BATmirror(b), 0));
BBPreleaseref(b->batCacheid);
b = s;
}
s = NULL;
}
bn = (*grpfunc)(b, g, e, s, tp, skip_nils, abort_on_error);
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
if (s)
BBPreleaseref(s->batCacheid);
if (bn == NULL) {
char *errbuf = GDKerrbuf;
char *s;
if (errbuf && *errbuf) {
if (strncmp(errbuf, "!ERROR: ", 8) == 0)
errbuf += 8;
if (strchr(errbuf, '!') == errbuf + 5) {
s = createException(MAL, malfunc, "%s", errbuf);
} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
s = createException(MAL, malfunc, "%s", s + 2);
} else {
s = createException(MAL, malfunc, "%s", errbuf);
}
*GDKerrbuf = 0;
return s;
}
throw(MAL, malfunc, OPERATION_FAILED);
}
*retval = bn->batCacheid;
BBPkeepref(bn->batCacheid);
return MAL_SUCCEED;
}
/* Load variable varid from data set ncid into the bat v. Generate dimension
* bats dim using NCDFARRAYseries */
static str
NCDFloadVar(bat **dim, bat *v, int ncid, int varid, nc_type vtype, int vndims, int *vdims)
{
BAT *res;
bat vbid, *dim_bids;
int retval, i, j;
char *sermsg = NULL;
size_t sz = 1;
size_t *dlen = NULL, *val_rep = NULL, *grp_rep = NULL;
if ( dim == NULL )
return createException(MAL, "netcdf.importvar", "array of dimension bat is NULL");
dim_bids = *dim;
dlen = (size_t *)GDKzalloc(sizeof(size_t) * vndims);
for (i = 0; i < vndims; i++){
if ((retval = nc_inq_dimlen(ncid, vdims[i], &dlen[i])))
return createException(MAL, "netcdf.importvar",
"Cannot read dimension %d : %s",
vdims[i], nc_strerror(retval));
sz *= dlen[i];
}
switch (vtype) {
case NC_INT:
{
LOAD_NCDF_VAR(int,int);
break;
}
case NC_FLOAT:
case NC_DOUBLE:
{
LOAD_NCDF_VAR(dbl,double);
break;
}
default:
return createException(MAL, "netcdf.importvar",
"Type %s not supported yet",
prim_type_name(vtype));
}
BATsetcount(res, sz);
res->T->nonil = TRUE;
res->T->nil = FALSE;
res->tsorted = FALSE;
res->trevsorted = FALSE;
BATkey(BATmirror(res), FALSE);
BBPkeepref(vbid = res->batCacheid);
res = NULL;
/* Manually create dimensions with range [0:1:dlen[i]] */
val_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
grp_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
/* compute the repetition factor inside of the series (val_rep) and of series (grp_rep) */
for (i = 0; i < vndims; i++) {
val_rep[i] = grp_rep[i] = 1;
for (j = 0; j < i; j++)
grp_rep[i] *= dlen[j];
for (j = i + 1; j < vndims; j++)
val_rep[i] *= dlen[j];
}
for (i = 0; i < vndims; i++) {
sermsg = NCDFARRAYseries(&dim_bids[i], 0, 1, dlen[i], val_rep[i], grp_rep[i]);
if (sermsg != MAL_SUCCEED) {
BBPdecref(vbid, 1); /* undo the BBPkeepref(vbid) above */
for ( j = 0; j < i; j++) /* undo log. ref of previous dimensions */
BBPdecref(dim_bids[j], 1);
GDKfree(dlen);
GDKfree(val_rep);
GDKfree(grp_rep);
return createException(MAL, "netcdf.loadvar", "Failed to create a dimension of variable %d", varid);
}
}
/* to do : is descriptor check of dim_bids is needed? */
GDKfree(dlen);
GDKfree(val_rep);
GDKfree(grp_rep);
*v = vbid;
return MAL_SUCCEED;
}
static str
AGGRsubgroupedExt(bat *retval1, bat *retval2, bat *bid, bat *gid, bat *eid, bat *sid,
int skip_nils, int abort_on_error, int tp,
BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
BAT *(*quantilefunc)(BAT *, BAT *, BAT *, BAT *, int, double, int, int),
bat *quantile,
const char *malfunc)
{
BAT *b, *g, *e, *s, *bn = NULL, *cnts, *q = NULL;
double qvalue;
/* one of grpfunc1, grpfunc2 and quantilefunc is non-NULL and the others are */
assert((grpfunc1 && grpfunc2 == NULL && quantilefunc == NULL) ||
(grpfunc1 == NULL && grpfunc2 && quantilefunc == NULL) ||
(grpfunc1 == NULL && grpfunc2 == NULL && quantilefunc) );
/* if retval2 is non-NULL, we must have grpfunc2 */
assert(retval2 == NULL || grpfunc2 != NULL);
b = BATdescriptor(*bid);
g = gid ? BATdescriptor(*gid) : NULL;
e = eid ? BATdescriptor(*eid) : NULL;
q = quantile ? BATdescriptor(*quantile) : NULL;
if (b == NULL || (gid != NULL && g == NULL) || (eid != NULL && e == NULL)) {
if (b)
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
}
if (tp == TYPE_any && (grpfunc1 == BATgroupmedian || quantilefunc == BATgroupquantile))
tp = b->ttype;
if (sid) {
s = BATdescriptor(*sid);
if (s == NULL) {
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
}
} else {
if (!BAThdense(b)) {
/* XXX backward compatibility code: ignore non-dense head, but
* only if no candidate list */
s = BATmirror(BATmark(BATmirror(b), 0));
BBPreleaseref(b->batCacheid);
b = s;
}
s = NULL;
}
if (grpfunc1)
bn = (*grpfunc1)(b, g, e, s, tp, skip_nils, abort_on_error);
if (quantilefunc) {
assert(BATcount(q)>0);
assert(q->ttype == TYPE_dbl);
qvalue = ((const double *)Tloc(q, BUNfirst(q)))[0];
if (qvalue < 0|| qvalue > 1) {
char *s;
s = createException(MAL, malfunc, "quantile value of %f is not in range [0,1]", qvalue);
return s;
}
bn = (*quantilefunc)(b, g, e, s, tp, qvalue, skip_nils, abort_on_error);
}
if (grpfunc2 && (*grpfunc2)(&bn, retval2 ? &cnts : NULL, b, g, e, s, tp, skip_nils, abort_on_error) == GDK_FAIL)
bn = NULL;
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
if (s)
BBPreleaseref(s->batCacheid);
if (bn == NULL) {
char *errbuf = GDKerrbuf;
char *s;
if (errbuf && *errbuf) {
if (strncmp(errbuf, "!ERROR: ", 8) == 0)
errbuf += 8;
if (strchr(errbuf, '!') == errbuf + 5) {
s = createException(MAL, malfunc, "%s", errbuf);
} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
s = createException(MAL, malfunc, "%s", s + 2);
} else {
s = createException(MAL, malfunc, "%s", errbuf);
}
*GDKerrbuf = 0;
return s;
}
throw(MAL, malfunc, OPERATION_FAILED);
}
*retval1 = bn->batCacheid;
BBPkeepref(bn->batCacheid);
if (retval2) {
*retval2 = cnts->batCacheid;
BBPkeepref(cnts->batCacheid);
}
return MAL_SUCCEED;
}
/*
* grouped aggregates
*/
static str
AGGRgrouped(bat *retval1, bat *retval2, BAT *b, BAT *g, BAT *e, int tp,
BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
BAT *(*quantilefunc)(BAT *, BAT *, BAT *, BAT *, int, double, int, int),
BAT *quantile,
int skip_nils,
const char *malfunc)
{
BAT *bn, *cnts = NULL, *t, *map;
double qvalue;
/* one of grpfunc1, grpfunc2 and quantilefunc is non-NULL and the others are */
assert((grpfunc1 != NULL && grpfunc2 == NULL && quantilefunc == NULL) ||
(grpfunc1 == NULL && grpfunc2 != NULL && quantilefunc == NULL) ||
(grpfunc1 == NULL && grpfunc2 == NULL && quantilefunc != NULL) );
/* if retval2 is non-NULL, we must have grpfunc2 */
assert(retval2 == NULL || grpfunc2 != NULL);
assert(quantile == NULL || quantilefunc != NULL);
if (b == NULL || g == NULL || e == NULL) {
if (b)
BBPreleaseref(b->batCacheid);
if (g)
BBPreleaseref(g->batCacheid);
if (e)
BBPreleaseref(e->batCacheid);
throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
}
if (tp == TYPE_any && (grpfunc1 == BATgroupmedian || quantilefunc == BATgroupquantile))
tp = b->ttype;
if (!BAThdense(b) || !BAThdense(g)) {
/* if b or g don't have a dense head, replace the head with a
* dense sequence */
t = BATjoin(BATmirror(b), g, MIN(BATcount(b), BATcount(g)));
BBPreleaseref(b->batCacheid);
BBPreleaseref(g->batCacheid);
b = BATmirror(BATmark(t, 0));
g = BATmirror(BATmark(BATmirror(t), 0));
BBPreleaseref(t->batCacheid);
}
if (b->hseqbase != g->hseqbase || BATcount(b) != BATcount(g)) {
/* b and g are not aligned: align them by creating a view on
* one or the other */
oid min; /* lowest common oid */
oid max; /* highest common oid */
min = b->hseqbase;
if (min < g->hseqbase)
min = g->hseqbase;
max = b->hseqbase + BATcount(b);
if (g->hseqbase + BATcount(g) < max)
max = g->hseqbase + BATcount(g);
if (b->hseqbase != min || b->hseqbase + BATcount(b) != max) {
if (min >= max)
min = max = b->hseqbase;
t = BATslice(b, BUNfirst(b) + (BUN) (min - b->hseqbase),
BUNfirst(b) + (BUN) (max - b->hseqbase));
BBPreleaseref(b->batCacheid);
b = t;
}
if (g->hseqbase != min || g->hseqbase + BATcount(g) != max) {
if (min >= max)
min = max = g->hseqbase;
t = BATslice(g, BUNfirst(g) + (BUN) (min - g->hseqbase),
BUNfirst(g) + (BUN) (max - g->hseqbase));
BBPreleaseref(g->batCacheid);
g = t;
}
}
if (!BAThdense(e)) {
/* if e doesn't have a dense head, renumber the group ids with
* a dense sequence at the cost of some left joins */
map = BATmark(e, 0); /* [gid,newgid(dense)] */
BBPreleaseref(e->batCacheid);
e = BATmirror(map); /* [newgid(dense),gid] */
t = BATleftjoin(g, map, BATcount(g)); /* [oid,newgid] */
BBPreleaseref(g->batCacheid);
g = t;
} else {
map = NULL;
}
if (grpfunc1)
bn = (*grpfunc1)(b, g, e, NULL, tp, skip_nils, 1);
if (quantilefunc) {
assert(BATcount(quantile)>0);
assert(quantile->ttype == TYPE_dbl);
qvalue = ((const double *)Tloc(quantile, BUNfirst(quantile)))[0];
if (qvalue < 0|| qvalue > 1) {
char *s;
s = createException(MAL, malfunc, "quantile value of %f is not in range [0,1]", qvalue);
return s;
}
bn = (*quantilefunc)(b, g, e, NULL, tp, qvalue, skip_nils, 1);
}
if (grpfunc2 && (*grpfunc2)(&bn, retval2 ? &cnts : NULL, b, g, e, NULL, tp, skip_nils, 1) == GDK_FAIL)
bn = NULL;
if (bn != NULL && (grpfunc1 == BATgroupmin || grpfunc1 == BATgroupmax)) {
t = BATproject(bn, b);
BBPreleaseref(bn->batCacheid);
bn = t;
}
BBPreleaseref(b->batCacheid);
BBPreleaseref(g->batCacheid);
if (map == NULL) /* if map!=NULL, e is mirror of map */
BBPreleaseref(e->batCacheid);
if (bn == NULL) {
char *errbuf = GDKerrbuf;
char *s;
if (map)
BBPreleaseref(map->batCacheid);
if (errbuf && *errbuf) {
if (strncmp(errbuf, "!ERROR: ", 8) == 0)
errbuf += 8;
if (strchr(errbuf, '!') == errbuf + 5) {
s = createException(MAL, malfunc, "%s", errbuf);
} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
s = createException(MAL, malfunc, "%s", s + 2);
} else {
s = createException(MAL, malfunc, "%s", errbuf);
}
*GDKerrbuf = 0;
return s;
}
throw(MAL, malfunc, OPERATION_FAILED);
}
if (map) {
t = BATleftjoin(map, bn, BATcount(bn));
BBPreleaseref(bn->batCacheid);
bn = t;
if (cnts) {
t = BATleftjoin(map, cnts, BATcount(cnts));
BBPreleaseref(cnts->batCacheid);
cnts = t;
}
BBPreleaseref(map->batCacheid);
}
*retval1 = bn->batCacheid;
BBPkeepref(bn->batCacheid);
if (retval2) {
*retval2 = cnts->batCacheid;
BBPkeepref(cnts->batCacheid);
}
return MAL_SUCCEED;
}
static str
MATpackSliceInternal(MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i, i1 = p->argc, i2 = -1, *ret = (int*) getArgReference(stk,p,0);
BAT *b, *bn;
BUN cap = 0, fst, lst, cnt, c;
int ht = TYPE_any, tt = TYPE_any;
assert(p->argc > 3);
switch getArgType(mb,p,1) {
case TYPE_wrd:
fst = (BUN) *(wrd*) getArgReference(stk,p,1);
break;
case TYPE_lng:
fst = (BUN) *(lng*) getArgReference(stk,p,1);
break;
case TYPE_int:
fst = (BUN) *(int*) getArgReference(stk,p,1);
break;
default:
throw(MAL, "mat.packSlice", "wrong type for lower bound");
}
switch getArgType(mb,p,2) {
case TYPE_wrd: {
wrd l = *(wrd*) getArgReference(stk,p,2);
if (l == wrd_nil)
lst = BUN_MAX; /* no upper bound */
else
lst = (BUN) l;
break;
}
case TYPE_lng: {
lng l = *(lng*) getArgReference(stk,p,2);
if (l == lng_nil)
lst = BUN_MAX; /* no upper bound */
else
lst = (BUN) l;
break;
}
case TYPE_int: {
int l = *(int*) getArgReference(stk,p,2);
if (l == int_nil)
lst = BUN_MAX; /* no upper bound */
else
lst = (BUN) l;
break;
}
default:
throw(MAL, "mat.packSlice", "wrong type for upper bound");
}
if (lst < BUN_MAX)
lst++; /* inclusive -> exclusive upper bound */
if (lst < fst)
lst = fst;
cnt = lst - fst;
for (i = 3; i < p->argc && cap < lst; i++) {
int bid = stk->stk[getArg(p,i)].val.ival;
b = BBPquickdesc(abs(bid),FALSE);
if (b && bid < 0)
b = BATmirror(b);
if (b == NULL)
throw(MAL, "mat.packSlice", RUNTIME_OBJECT_MISSING);
if (ht == TYPE_any){
ht = b->htype;
tt = b->ttype;
}
c = BATcount(b);
if (cap <= fst) {
/* The optimal case is when the requested slice falls completely in one BAT.
* In that case, we can simply return a slice (view) of that BAT.
* (A pitty that we have calculated the other slices as well.)
*/
if (lst <= cap + c) {
b = BATdescriptor(bid);
if( b){
bn = BATslice(b, fst - cap, lst - cap);
BBPunfix(b->batCacheid);
BBPkeepref(*ret = bn->batCacheid);
} else
throw(MAL, "mat.packSlice", RUNTIME_OBJECT_MISSING);
return MAL_SUCCEED;
}
if (fst < cap + c) {
/* fst falls in BAT i1 == i */
i1 = i;
fst -= cap;
lst -= cap;
cap = 0;
}
}
cap += c;
}
/* lst falls in BAT i2 == i-1 */
i2 = i - 1;
if (cap <= fst) /* i.e., (i1 > i2) */
cap = 0;
else
cap -= fst;
cnt = MIN(cnt, cap);
assert(ht== TYPE_void);
bn = BATnew(TYPE_void, tt, cnt, TRANSIENT);
if (bn == NULL)
throw(MAL, "mat.packSlice", MAL_MALLOC_FAIL);
/* must set seqbase or else BATins will not materialize column */
BATseqbase(bn, 0);
if (tt == TYPE_void)
BATseqbase(BATmirror(bn), 0);
for (i = i1; i <= i2; i++) {
b = BATdescriptor(stk->stk[getArg(p,i)].val.ival);
if (b == NULL){
BBPreleaseref(bn->batCacheid);
throw(MAL, "mat.packSlice", RUNTIME_OBJECT_MISSING);
}
c = BATcount(b);
/* use the right oid ranges, don't change the input */
if (i == i1 && fst > 0) {
BAT *bb = b;
b = BATslice(bb, fst, c);
BBPunfix(bb->batCacheid);
} else
if (i == i2 && lst < c) {
BAT *bb = b;
b = BATslice(bb, 0, lst);
BBPunfix(bb->batCacheid);
}
BATins(bn,b,FALSE);
lst -= c;
BBPunfix(b->batCacheid);
}
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
// merging multiple OID lists, optimized for empty bats
// Further improvement should come from multi-bat merging.
str
MATmergepack(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i,j= 0, *ret = (int*) getArgReference(stk,p,0);
int top=0;
oid **o_end, **o_src, *o, *oo, onxt;
BAT *b, *bn, *bm, **bats;
BUN cap = 0;
(void)cntxt;
(void)mb;
bats = (BAT**) GDKzalloc(sizeof(BAT*) * p->argc);
o_end = (oid**) GDKzalloc(sizeof(oid*) * p->argc);
o_src = (oid**) GDKzalloc(sizeof(oid*) * p->argc);
if ( bats ==0 || o_end == 0 || o_src == 0){
if (bats) GDKfree(bats);
if (o_src) GDKfree(o_src);
if (o_end) GDKfree(o_end);
throw(MAL,"mat.mergepack",MAL_MALLOC_FAIL);
}
for (i = 1; i < p->argc; i++) {
int bid = stk->stk[getArg(p,i)].val.ival;
b = BATdescriptor(abs(bid));
if (b ){
cap += BATcount(b);
if ( BATcount(b) ){
// pre-sort the arguments
onxt = *(oid*) Tloc(b,BUNfirst(b));
for( j =top; j > 0 && onxt < *o_src[j-1]; j--){
o_src[j] = o_src[j-1];
o_end[j] = o_end[j-1];
bats[j] = bats[j-1];
}
o_src[j] = (oid*) Tloc(b,BUNfirst(b));
o_end[j] = o_src[j] + BATcount(b);
bats[j] = b;
top++;
}
}
}
bn = BATnew(TYPE_void, TYPE_oid, cap, TRANSIENT);
if (bn == NULL){
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
}
if ( cap == 0){
BATseqbase(bn, 0);
BATseqbase(BATmirror(bn), 0);
BBPkeepref(*ret = bn->batCacheid);
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
return MAL_SUCCEED;
}
BATseqbase(bn, bats[0]->hseqbase);
// UNROLL THE MULTI-BAT MERGE
o = (oid*) Tloc(bn,BUNfirst(bn));
while( top){
*o++ = *o_src[0];
o_src[0]++;
if( o_src[0] == o_end[0]){
// remove this one
for(j=0; j< top; j++){
o_src[j]= o_src[j+1];
o_end[j]= o_end[j+1];
bats[j] = bats[j+1];
}
top--;
} else{
// resort priority queue
onxt= *o_src[0];
for( j=1; j< top && onxt > *o_src[j]; j++){
oo = o_src[j]; o_src[j]= o_src[j-1]; o_src[j-1]= oo;
oo = o_end[j]; o_end[j]= o_end[j-1]; o_end[j-1]= oo;
bm = bats[j]; bats[j]=bats[j-1]; bats[j-1] = bm;
}
}
}
for( i=0; i< top; i++)
BBPunfix(bats[i]->batCacheid);
BATsetcount(bn, (BUN) (o - (oid *) Tloc(bn, BUNfirst(bn))));
BATseqbase(bn, 0);
BATsettrivprop(bn);
GDKfree(bats);
GDKfree(o_src);
GDKfree(o_end);
/* properties */
bn->trevsorted = 0;
bn->tsorted = 1;
bn->tkey = 1;
bn->T->nil = 0;
bn->T->nonil = 1;
BBPkeepref(*ret = bn->batCacheid);
return MAL_SUCCEED;
}
