/*
* Destroy a view.
*/
void
VIEWdestroy(BAT *b)
{
assert(isVIEW(b));
/* remove any leftover private hash structures */
if (b->H->hash)
HASHremove(BATmirror(b));
if (b->T->hash)
HASHremove(b);
IMPSdestroy(b);
VIEWunlink(b);
if (b->htype && !b->H->heap.parentid) {
HEAPfree(&b->H->heap, 0);
} else {
b->H->heap.base = NULL;
b->H->heap.filename = NULL;
}
if (b->ttype && !b->T->heap.parentid) {
HEAPfree(&b->T->heap, 0);
} else {
b->T->heap.base = NULL;
b->T->heap.filename = NULL;
}
b->H->vheap = NULL;
b->T->vheap = NULL;
BATfree(b);
}
/*
* The prime routine for the BAT layer is to create a new hash index.
* Its argument is the element type and the maximum number of BUNs be
* stored under the hash function.
*/
BAT *
BAThash(BAT *b, BUN masksize)
{
BAT *o = NULL;
lng t0,t1;
(void) t0;
(void) t1;
if (VIEWhparent(b)) {
bat p = VIEWhparent(b);
o = b;
b = BATdescriptor(p);
if (!ALIGNsynced(o, b) || BUNfirst(o) != BUNfirst(b)) {
BBPunfix(b->batCacheid);
b = o;
o = NULL;
}
}
MT_lock_set(&GDKhashLock(ABS(b->batCacheid)), "BAThash");
if (b->H->hash == NULL) {
unsigned int tpe = ATOMstorage(b->htype);
BUN cnt = BATcount(b);
BUN mask;
BUN p = BUNfirst(b), q = BUNlast(b), r;
Hash *h = NULL;
Heap *hp = NULL;
str nme = BBP_physical(b->batCacheid);
BATiter bi = bat_iterator(b);
ALGODEBUG fprintf(stderr, "#BAThash: create hash(" BUNFMT ");\n", BATcount(b));
/* cnt = 0, hopefully there is a proper capacity from
* which we can derive enough information */
if (!cnt)
cnt = BATcapacity(b);
if (b->htype == TYPE_void) {
if (b->hseqbase == oid_nil) {
MT_lock_unset(&GDKhashLock(ABS(b->batCacheid)), "BAThash");
ALGODEBUG fprintf(stderr, "#BAThash: cannot create hash-table on void-NIL column.\n");
return NULL;
}
ALGODEBUG fprintf(stderr, "#BAThash: creating hash-table on void column..\n");
tpe = TYPE_void;
}
/* determine hash mask size p = first; then no dynamic
* scheme */
if (masksize > 0) {
mask = HASHmask(masksize);
} else if (ATOMsize(ATOMstorage(tpe)) == 1) {
mask = (1 << 8);
} else if (ATOMsize(ATOMstorage(tpe)) == 2) {
mask = (1 << 12);
} else if (b->hkey) {
mask = HASHmask(cnt);
} else {
/* dynamic hash: we start with
* HASHmask(cnt/64); if there are too many
* collisions we try HASHmask(cnt/16), then
* HASHmask(cnt/4), and finally
* HASHmask(cnt). */
mask = HASHmask(cnt >> 6);
p += (cnt >> 2); /* try out on first 25% of b */
if (p > q)
p = q;
}
if (mask < 1024)
mask = 1024;
t0 = GDKusec();
do {
BUN nslots = mask >> 3; /* 1/8 full is too full */
r = BUNfirst(b);
if (hp) {
HEAPfree(hp);
GDKfree(hp);
}
if (h) {
ALGODEBUG fprintf(stderr, "#BAThash: retry hash construction\n");
GDKfree(h);
}
/* create the hash structures */
hp = (Heap *) GDKzalloc(sizeof(Heap));
if (hp &&
(hp->filename = GDKmalloc(strlen(nme) + 12)) != NULL)
sprintf(hp->filename, "%s.%chash", nme, b->batCacheid > 0 ? 'h' : 't');
if (hp == NULL ||
hp->filename == NULL ||
(h = HASHnew(hp, ATOMtype(b->htype), BATcapacity(b), mask)) == NULL) {
MT_lock_unset(&GDKhashLock(ABS(b->batCacheid)), "BAThash");
if (hp != NULL) {
GDKfree(hp->filename);
GDKfree(hp);
}
return NULL;
}
switch (tpe) {
case TYPE_bte:
starthash(bte);
break;
case TYPE_sht:
starthash(sht);
break;
case TYPE_int:
case TYPE_flt:
starthash(int);
break;
case TYPE_dbl:
case TYPE_lng:
starthash(lng);
break;
default:
for (; r < p; r++) {
ptr v = BUNhead(bi, r);
BUN c = (BUN) heap_hash_any(b->H->vheap, h, v);
if ( HASHget(h,c) == HASHnil(h) &&
nslots-- == 0)
break; /* mask too full */
HASHputlink(h,r, HASHget(h,c));
HASHput(h,c, r);
}
break;
}
} while (r < p && mask < cnt && (mask <<= 2));
/* finish the hashtable with the current mask */
p = r;
switch (tpe) {
case TYPE_bte:
finishhash(bte);
break;
case TYPE_sht:
finishhash(sht);
break;
case TYPE_int:
case TYPE_flt:
finishhash(int);
break;
case TYPE_dbl:
case TYPE_lng:
finishhash(lng);
break;
default:
for (; p < q; p++) {
ptr v = BUNhead(bi, p);
BUN c = (BUN) heap_hash_any(b->H->vheap, h, v);
HASHputlink(h,p, HASHget(h,c));
HASHput(h,c,p);
}
break;
}
b->H->hash = h;
t1 = GDKusec();
ALGODEBUG
fprintf(stderr, "#BAThash: hash construction "LLFMT" usec\n", t1-t0);
ALGODEBUG HASHcollisions(b,b->H->hash);
}
gdk_return
BATmaterializeh(BAT *b)
{
int ht;
BUN cnt;
Heap head;
BUN p, q;
oid h, *x;
bte tshift;
BATcheck(b, "BATmaterialize", GDK_FAIL);
assert(!isVIEW(b));
ht = b->htype;
cnt = BATcapacity(b);
head = b->H->heap;
p = BUNfirst(b);
q = BUNlast(b);
assert(cnt >= q - p);
ALGODEBUG fprintf(stderr, "#BATmaterialize(%d);\n", (int) b->batCacheid);
if (!BAThdense(b) || ht != TYPE_void) {
/* no voids */
return GDK_SUCCEED;
}
ht = TYPE_oid;
/* cleanup possible ACC's */
HASHdestroy(b);
IMPSdestroy(b);
b->H->heap.filename = NULL;
if (HEAPalloc(&b->H->heap, cnt, sizeof(oid)) != GDK_SUCCEED) {
b->H->heap = head;
return GDK_FAIL;
}
/* point of no return */
b->htype = ht;
tshift = b->T->shift;
BATsetdims(b);
if (b->ttype) {
b->T->shift = tshift; /* restore in case it got changed */
b->T->width = 1 << tshift;
}
b->batDirty = TRUE;
b->batDirtydesc = TRUE;
b->H->heap.dirty = TRUE;
/* set the correct dense info */
b->hdense = TRUE;
/* So now generate [h..h+cnt-1] */
h = b->hseqbase;
x = (oid *) b->H->heap.base;
for (; p < q; p++)
*x++ = h++;
cnt = h - b->hseqbase;
BATsetcount(b, cnt);
/* cleanup the old heaps */
HEAPfree(&head, 0);
return GDK_SUCCEED;
}
/*
* Materialize a view into a normal BAT. If it is a slice, we really
* want to reduce storage of the new BAT.
*/
gdk_return
VIEWreset(BAT *b)
{
bat hp, tp, hvp, tvp;
Heap head, tail, hh, th;
BAT *n = NULL, *v = NULL;
if (b == NULL)
return GDK_FAIL;
hp = VIEWhparent(b);
tp = VIEWtparent(b);
hvp = VIEWvhparent(b);
tvp = VIEWvtparent(b);
if (hp || tp) {
BAT *m;
BATstore *bs;
BUN cnt;
str nme;
size_t nmelen;
/* alloc heaps */
memset(&head, 0, sizeof(Heap));
memset(&tail, 0, sizeof(Heap));
memset(&hh, 0, sizeof(Heap));
memset(&th, 0, sizeof(Heap));
n = BATdescriptor(abs(b->batCacheid)); /* normalized */
if (n == NULL)
goto bailout;
m = BATmirror(n); /* mirror of normalized */
bs = BBP_desc(n->batCacheid);
cnt = BATcount(n) + 1;
nme = BBP_physical(n->batCacheid);
nmelen = nme ? strlen(nme) : 0;
assert(n->batCacheid > 0);
assert(hp || !b->htype);
assert(tp || !b->ttype);
head.farmid = BBPselectfarm(n->batRole, n->htype, offheap);
tail.farmid = BBPselectfarm(n->batRole, n->ttype, offheap);
if (n->htype) {
head.filename = (str) GDKmalloc(nmelen + 12);
if (head.filename == NULL)
goto bailout;
snprintf(head.filename, nmelen + 12, "%s.head", nme);
if (n->htype && HEAPalloc(&head, cnt, Hsize(n)) != GDK_SUCCEED)
goto bailout;
}
if (n->ttype) {
tail.filename = (str) GDKmalloc(nmelen + 12);
if (tail.filename == NULL)
goto bailout;
snprintf(tail.filename, nmelen + 12, "%s.tail", nme);
if (n->ttype && HEAPalloc(&tail, cnt, Tsize(n)) != GDK_SUCCEED)
goto bailout;
}
if (n->H->vheap) {
hh.farmid = BBPselectfarm(n->batRole, n->htype, varheap);
hh.filename = (str) GDKmalloc(nmelen + 12);
if (hh.filename == NULL)
goto bailout;
snprintf(hh.filename, nmelen + 12, "%s.hheap", nme);
if (ATOMheap(n->htype, &hh, cnt) != GDK_SUCCEED)
goto bailout;
}
if (n->T->vheap) {
th.farmid = BBPselectfarm(n->batRole, n->ttype, varheap);
th.filename = (str) GDKmalloc(nmelen + 12);
if (th.filename == NULL)
goto bailout;
snprintf(th.filename, nmelen + 12, "%s.theap", nme);
if (ATOMheap(n->ttype, &th, cnt) != GDK_SUCCEED)
goto bailout;
}
v = VIEWcreate(n, n);
if (v == NULL)
goto bailout;
/* cut the link to your parents */
VIEWunlink(n);
if (hp) {
BBPunshare(hp);
BBPunfix(hp);
}
if (tp) {
BBPunshare(tp);
BBPunfix(tp);
}
if (hvp) {
BBPunshare(hvp);
BBPunfix(hvp);
}
if (tvp) {
BBPunshare(tvp);
BBPunfix(tvp);
}
/* make sure everything points there */
m->S = n->S = &bs->S;
m->T = n->H = &bs->H;
m->H = n->T = &bs->T;
n->H->type = v->H->type;
n->H->varsized = v->H->varsized;
n->H->shift = v->H->shift;
n->H->width = v->H->width;
n->H->seq = v->H->seq;
n->T->type = v->T->type;
n->T->varsized = v->T->varsized;
n->T->shift = v->T->shift;
n->T->width = v->T->width;
n->T->seq = v->T->seq;
n->H->heap.parentid = n->T->heap.parentid = 0;
n->batRestricted = BAT_WRITE;
/* reset BOUND2KEY */
n->H->key = BAThkey(v);
n->T->key = BATtkey(v);
/* copy the heaps */
n->H->heap = head;
n->T->heap = tail;
/* unshare from parents heap */
if (hh.base) {
assert(n->H->vheap == NULL);
n->H->vheap = (Heap *) GDKzalloc(sizeof(Heap));
if (n->H->vheap == NULL)
goto bailout;
*n->H->vheap = hh;
n->H->vheap->parentid = n->batCacheid;
}
if (th.base) {
assert(n->T->vheap == NULL);
n->T->vheap = (Heap *) GDKzalloc(sizeof(Heap));
if (n->T->vheap == NULL)
goto bailout;
*n->T->vheap = th;
n->T->vheap->parentid = n->batCacheid;
}
n->batSharecnt = 0;
n->batCopiedtodisk = 0;
n->batDirty = 1;
/* reset BOUND2KEY */
n->hkey = BAThkey(v);
n->tkey = BATtkey(v);
/* make the BAT empty and insert all again */
DELTAinit(n);
/* reset capacity */
n->batCapacity = cnt;
/* swap n and v in case the original input was reversed, because
* BATins demands (v)oid-headed input */
if (b->batCacheid < 0) {
n = m;
m = BATmirror(v);
} else {
m = v;
}
/* insert all of v in n, and quit */
BATins(n, m, FALSE);
BBPreclaim(v);
BBPunfix(n->batCacheid);
}
return GDK_SUCCEED;
bailout:
BBPreclaim(v);
if (n != NULL)
BBPunfix(n->batCacheid);
HEAPfree(&head, 0);
HEAPfree(&tail, 0);
HEAPfree(&hh, 0);
HEAPfree(&th, 0);
return GDK_FAIL;
}
/*
* The prime routine for the BAT layer is to create a new hash index.
* Its argument is the element type and the maximum number of BUNs be
* stored under the hash function.
*/
gdk_return
BAThash(BAT *b, BUN masksize)
{
BAT *o = NULL;
lng t0 = 0, t1 = 0;
if (BATcheckhash(b)) {
if (o != NULL) {
o->T->hash = b->T->hash;
BBPunfix(b->batCacheid);
}
return GDK_SUCCEED;
}
MT_lock_set(&GDKhashLock(abs(b->batCacheid)), "BAThash");
if (b->T->hash == NULL) {
unsigned int tpe = ATOMbasetype(b->ttype);
BUN cnt = BATcount(b);
BUN mask, maxmask = 0;
BUN p = BUNfirst(b), q = BUNlast(b), r;
Hash *h = NULL;
Heap *hp;
const char *nme = BBP_physical(b->batCacheid);
const char *ext = b->batCacheid > 0 ? "thash" : "hhash";
BATiter bi = bat_iterator(b);
#ifdef PERSISTENTHASH
int fd;
#endif
ALGODEBUG fprintf(stderr, "#BAThash: create hash(" BUNFMT ");\n", BATcount(b));
if ((hp = GDKzalloc(sizeof(*hp))) == NULL ||
(hp->farmid = BBPselectfarm(b->batRole, b->ttype, hashheap)) < 0 ||
(hp->filename = GDKmalloc(strlen(nme) + 12)) == NULL) {
MT_lock_unset(&GDKhashLock(abs(b->batCacheid)), "BAThash");
GDKfree(hp);
return GDK_FAIL;
}
sprintf(hp->filename, "%s.%s", nme, ext);
/* cnt = 0, hopefully there is a proper capacity from
* which we can derive enough information */
if (!cnt)
cnt = BATcapacity(b);
if (b->ttype == TYPE_void) {
if (b->tseqbase == oid_nil) {
MT_lock_unset(&GDKhashLock(abs(b->batCacheid)), "BAThash");
ALGODEBUG fprintf(stderr, "#BAThash: cannot create hash-table on void-NIL column.\n");
GDKfree(hp->filename);
GDKfree(hp);
return GDK_FAIL;
}
ALGODEBUG fprintf(stderr, "#BAThash: creating hash-table on void column..\n");
tpe = TYPE_void;
}
/* determine hash mask size p = first; then no dynamic
* scheme */
if (masksize > 0) {
mask = HASHmask(masksize);
} else if (ATOMsize(tpe) == 1) {
mask = (1 << 8);
} else if (ATOMsize(tpe) == 2) {
mask = (1 << 16);
} else if (b->tkey) {
mask = HASHmask(cnt);
} else {
/* dynamic hash: we start with
* HASHmask(cnt)/64; if there are too many
* collisions we try HASHmask(cnt)/16, then
* HASHmask(cnt)/4, and finally
* HASHmask(cnt). */
maxmask = HASHmask(cnt);
mask = maxmask >> 6;
p += (cnt >> 2); /* try out on first 25% of b */
if (p > q)
p = q;
}
t0 = GDKusec();
do {
BUN nslots = mask >> 3; /* 1/8 full is too full */
r = BUNfirst(b);
if (h) {
char *fnme;
bte farmid;
ALGODEBUG fprintf(stderr, "#BAThash: retry hash construction\n");
fnme = GDKstrdup(hp->filename);
farmid = hp->farmid;
HEAPfree(hp, 1);
memset(hp, 0, sizeof(*hp));
hp->filename = fnme;
hp->farmid = farmid;
GDKfree(h);
h = NULL;
}
/* create the hash structures */
if ((h = HASHnew(hp, ATOMtype(b->ttype), BATcapacity(b), mask, BATcount(b))) == NULL) {
MT_lock_unset(&GDKhashLock(abs(b->batCacheid)), "BAThash");
GDKfree(hp->filename);
GDKfree(hp);
return GDK_FAIL;
}
switch (tpe) {
case TYPE_bte:
starthash(bte);
break;
case TYPE_sht:
starthash(sht);
break;
case TYPE_int:
case TYPE_flt:
#if SIZEOF_OID == SIZEOF_INT
case TYPE_oid:
#endif
#if SIZEOF_WRD == SIZEOF_INT
case TYPE_wrd:
#endif
starthash(int);
break;
case TYPE_dbl:
case TYPE_lng:
#if SIZEOF_OID == SIZEOF_LNG
case TYPE_oid:
#endif
#if SIZEOF_WRD == SIZEOF_LNG
case TYPE_wrd:
#endif
starthash(lng);
break;
#ifdef HAVE_HGE
case TYPE_hge:
starthash(hge);
break;
#endif
default:
for (; r < p; r++) {
ptr v = BUNtail(bi, r);
BUN c = (BUN) heap_hash_any(b->T->vheap, h, v);
if (HASHget(h, c) == HASHnil(h) &&
nslots-- == 0)
break; /* mask too full */
HASHputlink(h, r, HASHget(h, c));
HASHput(h, c, r);
}
break;
}
} while (r < p && mask < maxmask && (mask <<= 2));
/* finish the hashtable with the current mask */
p = r;
switch (tpe) {
case TYPE_bte:
finishhash(bte);
break;
case TYPE_sht:
finishhash(sht);
break;
case TYPE_int:
case TYPE_flt:
#if SIZEOF_OID == SIZEOF_INT
case TYPE_oid:
#endif
#if SIZEOF_WRD == SIZEOF_INT
case TYPE_wrd:
#endif
finishhash(int);
break;
case TYPE_dbl:
case TYPE_lng:
#if SIZEOF_OID == SIZEOF_LNG
case TYPE_oid:
#endif
#if SIZEOF_WRD == SIZEOF_LNG
case TYPE_wrd:
#endif
finishhash(lng);
break;
#ifdef HAVE_HGE
case TYPE_hge:
finishhash(hge);
break;
#endif
default:
for (; p < q; p++) {
ptr v = BUNtail(bi, p);
BUN c = (BUN) heap_hash_any(b->T->vheap, h, v);
HASHputlink(h, p, HASHget(h, c));
HASHput(h, c, p);
}
break;
}
#ifdef PERSISTENTHASH
if ((BBP_status(b->batCacheid) & BBPEXISTING) &&
b->batInserted == b->batCount &&
HEAPsave(hp, nme, ext) == GDK_SUCCEED &&
(fd = GDKfdlocate(hp->farmid, nme, "rb+", ext)) >= 0) {
ALGODEBUG fprintf(stderr, "#BAThash: persisting hash %d\n", b->batCacheid);
((size_t *) hp->base)[0] |= 1 << 24;
if (write(fd, hp->base, SIZEOF_SIZE_T) < 0)
perror("write hash");
if (!(GDKdebug & FORCEMITOMASK)) {
#if defined(NATIVE_WIN32)
_commit(fd);
#elif defined(HAVE_FDATASYNC)
fdatasync(fd);
#elif defined(HAVE_FSYNC)
fsync(fd);
#endif
}
close(fd);
} else
ALGODEBUG fprintf(stderr, "#BAThash: NOT persisting hash %d\n", b->batCacheid);
#endif
b->T->hash = h;
t1 = GDKusec();
ALGODEBUG fprintf(stderr, "#BAThash: hash construction " LLFMT " usec\n", t1 - t0);
ALGODEBUG HASHcollisions(b, b->T->hash);
}
