HXRET
_hxcheckbuf(HXLOCAL const*locp, HXBUF const*bufp)
{
HXFILE *hp = locp->file;
if (IS_MAP(hp, bufp->pgno)) {
if (!(bufp->data[0] & 1))
return bad_map_self;
if (bufp->pgno) { // extended map page.
if (bufp->next || bufp->used)
return bad_map_head;
} else { // root page.
if (bufp->used >= DATASIZE(hp) || bufp->used != hp->uleng)
return bad_map_head;
// Todo: check that HXROOT{pgsize,version} are good.
}
int lastbit, pos;
PAGENO lastmap = locp->npages - 1;
lastmap = _hxmap(hp, lastmap - lastmap % HXPGRATE, &lastbit);
if (bufp->pgno == lastmap) { // All bits beyond lastbit must be zero.
BYTE mask = -2 << (lastbit & 7);
for (pos = lastbit >> 3; pos < (int)DATASIZE(hp); ++pos, mask = -1)
if (bufp->data[pos] & mask)
return bad_overmap;
}
} else { // DATA page.
static HXRET
maps(HXFILE * hp)
{
HXLOCAL loc;
PAGENO pg, last;
int bitpos;
# define locp (&loc)
# define bufp (&loc.buf[0])
ENTER(locp, hp, NULL, 1);
locp->mode = F_RDLCK;
_hxlock(locp, 0, 0);
_hxsize(locp);
last = locp->npages - 1;
last = _hxmap(hp, last - last % HXPGRATE, &bitpos);
printf("last: mpg=%u bit=%d\n", (unsigned)last, bitpos);
for (pg = 0; pg <= last;) {
int i, n;
_hxload(locp, bufp, pg);
n = pg == last ? (bitpos + 7) >> 3 : (int)DATASIZE(hp);
printf("---- pgno=%u next=%u used=%d recs=%u", pg, bufp->next,
bufp->used, bufp->recs);
for (i = 0; i < n; ++i) {
unsigned b = bufp->data[i] & 0xFF;
if (!(i & 15))
printf("\n%7u", (unsigned)pg + i * 8 * HXPGRATE);
if (i < bufp->used)
fputs(" >>", stdout);
else
printf(" %02X", b);
}
putchar('\n');
pg += (DATASIZE(hp) - bufp->used) * 8 * HXPGRATE;
}
LEAVE(locp, 0);
# undef locp
}
int
main(int argc, char **argv)
{
int count = argc > 1 ? atoi(argv[1]) : 10000;
plan_tests(9);
ok(sizeof(off_t) == 8, "sizeof(off_t) == %" FSIZE "d", sizeof(off_t));
// TODO: add tests that actually test hx() on large databases
int npages = 1 << 20;
int pgsize = 8192;
int rc = hxcreate("large_t.hx", 0666, pgsize, NULL, 0);
ok(rc == HXOKAY, "hxcreate(large_t.hx): %d %s", rc, hxerror(rc));
HXFILE *hp = hxopen("large_t.hx", HX_UPDATE);
ok(hp != NULL, "hxopen(large_t.hx): %d %s", errno, errname[errno]);
int fd = hxfileno(hp);
off_t size = (off_t) pgsize * npages; // 8GB
ok(!ftruncate(fd, size), "grow to %" FOFF "d: %d %s", size, errno,
errname[errno]);
// Manually initialize map pages.
// A map page must indicate that it itself (i.e. bit 0 in the map) is
// allocated, else it will eventually be allocated and overwritten as
// an overflow page.
// 32:8(bits/byte)*4(pgrate) pgsize:bytes/page 6:(page header overhead)
int pg, last = _hxmap(hp, npages, &pg); // "pg" is just junk here.
char map = 1;
for (pg = 0; (pg = NEXT_MAP(hp, pg)) <= last;) {
lseek(fd, (off_t) pg * pgsize + sizeof(HXPAGE), 0);
write(fd, &map, 1);
}
struct stat sb;
ok(!fstat(hxfileno(hp), &sb), "fstat: %d %s", errno, errname[errno]);
ok(sb.st_size == size, "size is 0x%" FOFF "x", sb.st_size);
// 10000 records is enough to ensure that at least SOME records end up
// above the 4GB mark.
int i, tally = 0;
char rec[11] = { };
for (i = rc = 0; i < count; ++i) {
sprintf(rec, "%08d", i);
rc = hxput(hp, rec, 10);
if (rc != 0)
break;
rc = hxget(hp, rec, 10);
if (rc != 10) {
fprintf(stderr, "hxget(%s) failed: %d\n", rec, rc);
break;
}
}
ok(i == count, "inserted and retrieved %d/%d records: %s", i, count,
hxerror(rc));
while (0 < (rc = hxnext(hp, rec, 11)))
++tally;
ok(tally == count, "hxnext retrieved %d/%d records: %s", tally, count,
hxerror(rc));
rc = hxfix(hp, 0, 0, 0, 0);
ok(rc == (HXRET) HX_UPDATE, "hxcheck: large_t.hx is ready for %s access",
hxmode(rc));
hxclose(hp);
return exit_status();
}
//--------------|---------------------------------------------
HXRET
hxshape(HXFILE * hp, double overload)
{
HXLOCAL loc, *locp = &loc;
HXBUF *srcp, *dstp, *oldp;
int pos, bitpos;
PGINFO *atail, *ztail;
PAGENO pg, pm, *aprev, *afree, *zfree;
double totbytes = 0, fullbytes = 0, fullpages = 0;
if (!hp || hp->buffer.pgno || hp->mode & HX_MMAP
|| !(hp->mode & HX_UPDATE))
return HXERR_BAD_REQUEST;
ENTER(locp, hp, NULL, 3);
_hxlock(locp, 0, 0);
_hxsize(locp);
srcp = &locp->buf[0];
dstp = &locp->buf[1];
oldp = &locp->buf[2];
_hxinitRefs(locp);
// Populate vnext,vrefs,vtail for tail-merging:
ztail = calloc(locp->npages / HXPGRATE + 1, sizeof(PGINFO));
locp->vtail = ztail;
for (pg = 1; pg < locp->npages; ++pg) {
PGINFO x = _hxpginfo(locp, pg);
_hxsetRef(locp, pg, x.pgno);
totbytes += x.used;
if (x.pgno) // i.e. page.next != 0
++fullpages, fullbytes += x.used;
else if (x.used && !IS_HEAD(pg))
x.pgno = pg, *ztail++ = x;
}
// Sort vtail by (used), so that smallest+largest (used)
// counts can be matched up; simple greedy-fill algorithm.
qsort(locp->vtail, ztail - locp->vtail,
sizeof *locp->vtail, (cmpfn_t) cmpused);
// Combine tail pages where possible:
for (atail = locp->vtail, --ztail; atail < ztail;) {
if (!_FITS(hp, atail->used, atail->recs, ztail->used, ztail->recs)) {
--ztail;
continue;
}
// Merge is always from [atail] to [ztail], to maintain
// ([ztail].used >= [ztail-1].used).
if (atail->pgno < ztail->pgno) {
PGINFO tmp = *atail;
*atail = *ztail;
*ztail = tmp;
}
_hxload(locp, srcp, atail->pgno);
_hxload(locp, dstp, ztail->pgno);
_hxappend(dstp, srcp->data, srcp->used);
dstp->recs += srcp->recs;
_hxsave(locp, dstp);
_hxfindRefs(locp, srcp, srcp->pgno);
for (aprev = locp->vprev; *aprev; ++aprev)
PUTLINK(locp, *aprev, dstp->pgno);
ztail->used += srcp->used;
srcp->used = srcp->recs = 0;
BUFLINK(locp, srcp, 0);
_hxsave(locp, srcp);
_hxalloc(locp, srcp->pgno, 0);
++atail;
}
// Now decide whether to grow or shrink the file.
PAGENO overflows = 0;
for (pg = 1; pg < locp->npages; ++pg) {
if (IS_HEAD(pg)) {
int loops = HX_MAX_CHAIN;
for (pm = pg; (pm = locp->vnext[pm]); ++overflows)
if (!--loops)
LEAVE(locp, HXERR_BAD_FILE);
}
}
PAGENO dpages = locp->dpages + overflows;
PAGENO goodsize = dpages / (1.0 + overload);
DEBUG("%.0f/%.0f=%0.f %.0f %lu/%.2f=%lu => %lu",
fullbytes, fullpages, fullbytes / fullpages,
totbytes, dpages, overload + 1, goodsize, _hxd2f(goodsize));
// "+1" for the root page
goodsize = goodsize ? _hxd2f(goodsize) + 1 : 2;
if (locp->npages <= goodsize) {
// Increase dpages.
// Note that _hxgrow always returns an ALLOCATED
// overflow. It would be smarter to clear all the
// map bits in one step at the end, the way
// hxbuild sets all the map bits in one load/save.
PAGENO junk = 0;
while (locp->npages < goodsize) {
_hxgrow(locp, dstp, DATASIZE(hp), &junk);
_hxsave(locp, dstp);
_hxalloc(locp, dstp->pgno, 0);
}
_hxflushfreed(locp, dstp);
LEAVE(locp, HXNOTE);
}
// Build a list of free pgnos
assert(sizeof *afree <= sizeof *locp->vtail);
afree = zfree = (PAGENO *) locp->vtail;
for (pg = pm = 0; pg < locp->npages; pg += HXPGRATE) {
if (!VREF(locp, pg) && !IS_MAP(hp, pg))
*zfree++ = pg;
}
// Since we are decrementing npages BEFORE reading last page,
// set locked such that _hxislocked gives correct answer.
hp->locked |= LOCKED_BEYOND;
// Work backward from end of file, trimming pages.
for (; locp->npages > goodsize; --locp->npages) {
PAGENO srchead = locp->npages - 1;
PAGENO srctail, dsthead, dstneck, dsttail;
int loops = HX_MAX_CHAIN;
// EASIEST CASE: a map or unreferenced oveflow page
if (srchead == zfree[-1] || IS_MAP(hp, srchead)) {
assert(!VREF(locp, srchead) && !IS_HEAD(srchead));
--zfree;
continue;
}
// EASIER CASE: an empty head page
_hxload(locp, srcp, srchead);
if (!srcp->used)
continue;
// Anything from here on might need 2 free pages
if (zfree - afree < 2)
break;
--VREF(locp, srcp->next);
STAIN(srcp);
srcp->pgno = *afree++;
# if 0
// hxshape does not work with MMAP as yet.
if (hp->mmap)
memcpy(&hp->mmap[(off_t) srcp->pgno * hp->pgsize],
srcp->page, hp->pgsize);
# endif
_hxalloc(locp, srcp->pgno, 1);
_hxsetRef(locp, srcp->pgno, srcp->next);
// EASY CASE: an overflow page to relocate:
if (!IS_HEAD(srchead)) {
_hxsave(locp, srcp);
_hxfindRefs(locp, srcp, srchead);
for (aprev = locp->vprev; *aprev; ++aprev)
PUTLINK(locp, *aprev, srcp->pgno);
continue;
}
// HARD CASE: a head page to desplit:
locp->hash = RECHASH(srcp->data);
_hxpoint(locp); // recalc (dpages,head)
dsthead = locp->head;
dstneck = locp->vnext[dsthead];
dsttail = _hxgetRef(locp, dsthead, 0);
srctail = _hxgetRef(locp, srchead, 0);
// Append srchead to dsttail, or insert chain between
// dsthead and vnext[dsthead]. If srctail is shared,
// make a copy of * it first.
if (dsttail == dsthead || VREF(locp, dsttail) == 1) {
dsthead = dsttail;
} else if (srctail == srchead) {
BUFLINK(locp, srcp, dstneck);
} else if (VREF(locp, srctail) == 1) {
PUTLINK(locp, srctail, dstneck);
} else if (srctail == dsttail) {
if (dsttail != dstneck) {
srctail = _hxgetRef(locp, srcp->pgno, srctail);
if (srctail == srcp->pgno) {
BUFLINK(locp, srcp, dstneck);
} else {
PUTLINK(locp, srctail, dstneck);
}
}
} else {
_hxload(locp, oldp, srctail);
if (!oldp->used)
LEAVE(locp, HXERR_BAD_FILE);
_hxfresh(locp, dstp, *afree++);
_hxalloc(locp, dstp->pgno, 1);
_hxshift(locp, locp->head, srchead, oldp, dstp, dstp);
_hxsave(locp, oldp);
// This hack prevents the CHECK in _hxlink from
// aborting when oldp contains a page that the
// next iteration wants to PUTLINK. This ONLY occurs
// in this code (I think).
// TODO: can this happen in (hxfix,hxshape)??
oldp->pgno = -1;
BUFLINK(locp, dstp, dstneck);
_hxsave(locp, dstp);
if (srcp->next == srctail) {
BUFLINK(locp, srcp, dstp->pgno);
} else {
pg = _hxgetRef(locp, srchead, srctail);
PUTLINK(locp, pg, dstp->pgno);
}
}
_hxload(locp, dstp, dsthead);
BUFLINK(locp, dstp, srcp->pgno);
// Cannot early-out on !SHRUNK here (as "hxput" does);
// new chain may have vacancies in two places.
while (1) {
if (!--loops)
LEAVE(locp, HXERR_BAD_FILE);
if (_hxshift(locp, locp->head, srchead, srcp, dstp, dstp)) {
SWAP(srcp, dstp);
} else {
BUFLINK(locp, dstp, srcp->next);
if (!srcp->used != !VREF(locp, srcp->pgno))
LEAVE(locp, HXERR_BAD_FILE);
if (!srcp->used) {
BUFLINK(locp, srcp, 0);
*--afree = srcp->pgno;
_hxalloc(locp, srcp->pgno, 0);
}
}
_hxsave(locp, srcp);
if (!dstp->next)
break;
_hxload(locp, srcp, dstp->next);
}
_hxsave(locp, dstp);
}
// npages was overdecremented by one in loop
_hxresize(locp, locp->npages + 1);
// Zero the freemap for all truncated overflow pages:
pg = _hxmap(hp, locp->npages + HXPGRATE
- locp->npages % HXPGRATE, &bitpos);
_hxload(locp, dstp, pg);
DEBUG2("clear map %lu from bit %d onward", pg, bitpos);
pos = bitpos >> 3;
dstp->data[pos++] &= ~(-1 << (bitpos & 7));
memset(dstp->data + pos, 0, DATASIZE(hp) - pos);
STAIN(dstp);
_hxsave(locp, dstp);
LEAVE(locp, HXOKAY);
}
