static int
dump(HXFILE * hp, FILE * fp)
{
HXLOCAL loc;
# define locp (&loc)
ENTER(locp, hp, NULL, 1);
locp->mode = F_RDLCK;
_hxlock(locp, 0, 0);
_hxsize(locp);
HXBUF *bufp = &locp->buf[0];
PAGENO pg;
int uleng = hxinfo(hp, NULL, 0);
char udata[uleng];
hxinfo(hp, udata, uleng);
locp->mode = F_RDLCK;
for (pg = 0; pg < locp->npages; ++pg) {
_hxload(locp, bufp, pg);
_hxprbuf(locp, bufp, fp);
}
LEAVE(locp, 0);
# undef locp
}
HXRET
hxstat(HXFILE * hp, HXSTAT * sp)
{
HXLOCAL loc, *locp = &loc;
PAGENO pg;
int i, nchains;
if (!hp || !sp)
return HXERR_BAD_REQUEST;
ENTER(locp, hp, 0, 1);
HXBUF *bufp = &locp->buf[0];
locp->mode = F_RDLCK;
_hxlock(locp, 0, 0);
_hxsize(locp);
memset(sp, 0, sizeof *sp);
sp->npages = locp->npages;
_hxinitRefs(locp);
for (pg = 1; pg < (unsigned)locp->npages; ++pg) {
_hxload(locp, bufp, pg);
if (IS_MAP(hp, pg))
continue;
_hxsetRef(locp, pg, bufp->next);
if (IS_HEAD(bufp->pgno)) {
sp->head_bytes += bufp->used;
} else if (bufp->used) {
++sp->ovfl_pages;
sp->ovfl_bytes += bufp->used;
}
char *recp, *endp;
FOR_EACH_REC(recp, bufp, endp) {
++sp->nrecs;
sp->hash ^= RECHASH(recp);
}
if (!IS_HEAD(pg)) {
COUNT nheads = 0;
_hxfindHeads(locp, bufp);
while (locp->vprev[nheads])
++nheads;
if (nheads > HX_MAX_SHARE)
nheads = HX_MAX_SHARE;
++sp->share_hist[nheads];
}
}
static int
hdrs(HXFILE * hp)
{
HXLOCAL loc;
# define locp (&loc)
PAGENO pg;
HXBUF *bufp;
ENTER(locp, hp, NULL, 1);
locp->mode = F_RDLCK;
_hxlock(locp, 0, 0);
_hxsize(locp);
bufp = &locp->buf[0];
printf("npages:%u dpages:%u mask:%u pgsize:%d"
" udata[%d]:'%.*s'\n",
locp->npages, locp->dpages, locp->mask,
hp->pgsize, hp->uleng, hp->uleng, hp->udata);
_hxload(&loc, bufp, 0);
dx(stdout, bufp->data, ((loc.npages + 31) >> 5) + 15);
locp->vprev = calloc(DATASIZE(locp->file) / MINRECSIZE, sizeof(PAGENO));
for (pg = 1; pg < loc.npages; ++pg) {
PAGENO *pp = locp->vprev;
_hxload(&loc, &loc.buf[0], pg);
_hxfindHeads(locp, bufp);
printf("%7u next=%u used=%d recs=%u heads:", (unsigned)pg,
(unsigned)bufp->next, bufp->used, bufp->recs);
while (*pp)
printf(" %u", *pp++);
putchar('\n');
}
LEAVE(locp, 0);
#undef locp
}
// (size < 0) means hxhold: head is left locked on exit.
// Usual usage is: hxhold(), modify record, hxput().
int
hxget(HXFILE * hp, char *rp, int size)
{
HXLOCAL loc, *locp = &loc;
int leng, rpos, loops, junk;
HXBUF *bufp;
char *recp;
if (!hp || !rp)
return HXERR_BAD_REQUEST;
if (hxdebug > 1) {
char buf[25] = {};
hx_save(hp, rp, size<0 ? -size-1 : size, buf, sizeof buf);
DEBUG2("%s('%s...')", size < 0 ? "hold" : "get",
buf, size);
}
ENTER(locp, hp, rp, 1);
if (size >= 0) {
locp->mode = F_RDLCK;
_hxlockset(locp, HEAD_LOCK); // just lock head
} else {
_hxlockset(locp, HIGH_LOCK); // lock head, +split if > head
HOLD_HEAD(locp);
}
if (IS_MMAP(hp)) _hxremap(locp);
leng = 0;
loops = HX_MAX_CHAIN;
bufp = &locp->buf[0];
bufp->next = locp->head;
do {
if (!--loops)
LEAVE(locp, HXERR_BAD_FILE);
_hxload(locp, bufp, bufp->next);
rpos = _hxfind(locp, bufp, locp->hash, rp, &junk);
} while (rpos < 0 && bufp->next);
if (rpos < 0)
LEAVE(locp, 0);
recp = bufp->data + rpos;
leng = RECLENG(recp);
memcpy(rp, RECDATA(recp), IMIN(leng, size<0? -size-1 : size));
LEAVE(locp, leng);
}
static HXRET
maps(HXFILE * hp)
{
HXLOCAL loc;
PAGENO pg, last;
int bitpos;
# define locp (&loc)
# define bufp (&loc.buf[0])
ENTER(locp, hp, NULL, 1);
locp->mode = F_RDLCK;
_hxlock(locp, 0, 0);
_hxsize(locp);
last = locp->npages - 1;
last = _hxmap(hp, last - last % HXPGRATE, &bitpos);
printf("last: mpg=%u bit=%d\n", (unsigned)last, bitpos);
for (pg = 0; pg <= last;) {
int i, n;
_hxload(locp, bufp, pg);
n = pg == last ? (bitpos + 7) >> 3 : (int)DATASIZE(hp);
printf("---- pgno=%u next=%u used=%d recs=%u", pg, bufp->next,
bufp->used, bufp->recs);
for (i = 0; i < n; ++i) {
unsigned b = bufp->data[i] & 0xFF;
if (!(i & 15))
printf("\n%7u", (unsigned)pg + i * 8 * HXPGRATE);
if (i < bufp->used)
fputs(" >>", stdout);
else
printf(" %02X", b);
}
putchar('\n');
pg += (DATASIZE(hp) - bufp->used) * 8 * HXPGRATE;
}
LEAVE(locp, 0);
# undef locp
}
//--------------|---------------------------------------------
HXRET
hxput(HXFILE * hp, char const *recp, int leng)
{
HXLOCAL loc, *locp = &loc;
if (!hp || leng < 0 || !recp || leng > hxmaxrec(hp)
|| !(hp->mode & HX_UPDATE) || !hp->test)
return HXERR_BAD_REQUEST;
if (leng && !hx_test(hp, recp, leng))
return HXERR_BAD_RECORD;
if (SCANNING(hp) && hx_diff(hp, recp, RECDATA(_hxcurrec(hp))))
return HXERR_BAD_REQUEST;
ENTER(locp, hp, recp, 3);
_hxlockset(locp, leng ? HIGH_LOCK : HEAD_LOCK);
if (IS_MMAP(hp))
_hxremap(locp);
int may_find = 1, loops = HX_MAX_CHAIN;
int newsize = leng ? leng + sizeof(HXREC) : 0;
HXBUF *currp = &locp->buf[0], *prevp = &locp->buf[1];
// If scanning is on an overflow page, and hxdel might
// empty the page, hxput after hxnext can't just jump to
// the right page, because (prevp) is not loaded,
// so deleting currp would hard.
_hxload(locp, currp, SCANNING(hp) && (leng || IS_HEAD(hp->buffer.pgno))
? hp->buffer.pgno : locp->head);
while (1) {
int pos, hindpos, skip = 0;
PAGENO nextpg = currp->next;
if (!--loops)
LEAVE(locp, HXERR_BAD_FILE);
// Search for the key (an old record to be deleted).
// If SCANNING: the file is locked, and the matching
// record must be there.
pos = !may_find ? -1
: !SCANNING(hp) ? _hxfind(locp, currp, locp->hash, recp, &hindpos)
: currp->pgno == hp->buffer.pgno ? hp->currpos : -1;
if (pos >= 0) {
char *oldp = currp->data + pos;
COUNT oldsize = RECSIZE(oldp);
int delta = newsize - oldsize;
locp->ret = RECLENG(oldp);
may_find = 0;
assert(!currp->delta);
currp->delpos = pos;
currp->delta = delta;
if (!newsize) { // hxdel or remove after inserted previously.
_hxremove(currp, pos, oldsize);
currp->recs--;
if (SCANNING(hp))
hp->recsize = 0;
} else if (FITS(hp, currp, delta, 0)) { // replace
if (delta) {
memmove(oldp + newsize, oldp + oldsize,
currp->used - pos - oldsize);
currp->used += delta;
STSH(leng, oldp + sizeof(PAGENO));
if (SCANNING(hp))
hp->recsize = newsize;
DEINDEX(currp); // force indexify
}
memcpy(oldp + sizeof(HXREC), recp, leng);
STAIN(currp);
newsize = 0;
} else if (SCANNING(hp)) {
// At this point we are stuck: if we delete the old copy of
// the record, we are committed to inserting the new copy
// somewhere else, but that might require changing links
// or even growing the file: a NO-NO during a hxnext scan.
LEAVE(locp, HXERR_BAD_REQUEST);
} else { // Delete old version and continue (insert elsewhere).
_hxremove(currp, pos, oldsize);
currp->recs--;
}
}
if (currp->used && !IS_HEAD(currp->pgno) && SHRUNK(prevp))
skip = !_hxshift(locp, locp->head, 0, currp, prevp, NULL);
// Insert the new record if it fits.
if (newsize && FITS(hp, currp, newsize, 1)) {
HXREC hdr;
STLG(locp->hash, &hdr.hash);
STSH(leng, &hdr.leng);
_hxappend(currp, (char *)&hdr, sizeof hdr);
_hxappend(currp, recp, leng);
currp->recs++;
newsize = 0;
}
// If the current page contains only data of OTHER heads
// -- and hence, must be at the END of a chain --
// unlink it from this chain. If the page is empty,
// unlink it AND put it in the freemap.
if (IS_HEAD(currp->pgno)) {
skip = 0;
} else if (!currp->used) {
skip = 1;
_hxputfreed(locp, currp);
if (SCANNING(hp) && hp->buffer.pgno == currp->pgno)
hp->buffer.used = 0;
} else if (currp->next || !SHRUNK(currp)) {
skip = 0;
} else if (!skip) { // If skip not set by _hxshift above...
char const *rp, *ep;
FOR_EACH_REC(rp, currp, ep)
if (locp->head == _hxhead(locp, RECHASH(rp)))
break;
skip = rp == ep; // No recs for locp->head in this tail.
}
if (skip)
LINK(prevp, nextpg);
else
SWAP(prevp, currp);
sync_save(locp, currp);
if (!newsize && !prevp->next)
break;
if (!newsize && !may_find && !SHRUNK(prevp))
break;
if (prevp->next) {
_hxload(locp, currp, prevp->next);
continue;
}
// We are at the end of the chain, and rec not yet inserted.
// Unlocking is necessary even if tail is not shared;
// it may be hp->tail.pgno in some other process.
if (!FILE_HELD(hp) && !IS_HEAD(prevp->pgno))
_hxunlock(locp, prevp->pgno, 1);
// _hxshare/_hxfindfree may update the map (root etc).
// Split MUST be locked before root, else risk deadlock.
_hxlockset(locp, BOTH_LOCK);
if (IS_MMAP(hp))
_hxremap(locp);
// At this point assert:
// - head is locked, split is locked,
// - head matches hash, npages matches filesize.
// After locking the split, no other process can change
// the file size.
may_find = 0;
COUNT need = IS_HEAD(prevp->pgno) ? newsize : 0;
if (!_hxshare(locp, currp, need)
&& !_hxgetfreed(locp, currp)
&& !_hxfindfree(locp, currp)) {
// _hxgrow will zero samehead if it splits locp->head.
PAGENO samehead = locp->head;
// _hxgrow will change the file length. A concurrent
// hxget/hxdel could miscalculate locp->head as
// being the newly-added page.
_hxlock(locp, locp->npages, 0);
_hxgrow(locp, currp, need, &samehead);
DEBUG3("head=%u samehead=%u", locp->head, samehead);
if (!samehead) {
_hxputfreed(locp, currp);
_hxpoint(locp);
_hxload(locp, currp, locp->head);
loops = HX_MAX_CHAIN;
continue;
}
}
// _hxgrow may clobber prevp, so we reload it. Even if
// prevp->pgno == locp->head, prevp may contain an
// obsolete copy of the head page. The empty page is
// always appended to head. _hxshare only returns true
// if currp is head and currp->next is 0, so it can't
// clobber it.
_hxsave(locp, prevp);
_hxload(locp, prevp, locp->head);
LINK(currp, prevp->next);
LINK(prevp, currp->pgno);
currp->orig = DATASIZE(hp); // make SHRUNK be true
}
//--------------|---------------------------------------------
HXRET
hxshape(HXFILE * hp, double overload)
{
HXLOCAL loc, *locp = &loc;
HXBUF *srcp, *dstp, *oldp;
int pos, bitpos;
PGINFO *atail, *ztail;
PAGENO pg, pm, *aprev, *afree, *zfree;
double totbytes = 0, fullbytes = 0, fullpages = 0;
if (!hp || hp->buffer.pgno || hp->mode & HX_MMAP
|| !(hp->mode & HX_UPDATE))
return HXERR_BAD_REQUEST;
ENTER(locp, hp, NULL, 3);
_hxlock(locp, 0, 0);
_hxsize(locp);
srcp = &locp->buf[0];
dstp = &locp->buf[1];
oldp = &locp->buf[2];
_hxinitRefs(locp);
// Populate vnext,vrefs,vtail for tail-merging:
ztail = calloc(locp->npages / HXPGRATE + 1, sizeof(PGINFO));
locp->vtail = ztail;
for (pg = 1; pg < locp->npages; ++pg) {
PGINFO x = _hxpginfo(locp, pg);
_hxsetRef(locp, pg, x.pgno);
totbytes += x.used;
if (x.pgno) // i.e. page.next != 0
++fullpages, fullbytes += x.used;
else if (x.used && !IS_HEAD(pg))
x.pgno = pg, *ztail++ = x;
}
// Sort vtail by (used), so that smallest+largest (used)
// counts can be matched up; simple greedy-fill algorithm.
qsort(locp->vtail, ztail - locp->vtail,
sizeof *locp->vtail, (cmpfn_t) cmpused);
// Combine tail pages where possible:
for (atail = locp->vtail, --ztail; atail < ztail;) {
if (!_FITS(hp, atail->used, atail->recs, ztail->used, ztail->recs)) {
--ztail;
continue;
}
// Merge is always from [atail] to [ztail], to maintain
// ([ztail].used >= [ztail-1].used).
if (atail->pgno < ztail->pgno) {
PGINFO tmp = *atail;
*atail = *ztail;
*ztail = tmp;
}
_hxload(locp, srcp, atail->pgno);
_hxload(locp, dstp, ztail->pgno);
_hxappend(dstp, srcp->data, srcp->used);
dstp->recs += srcp->recs;
_hxsave(locp, dstp);
_hxfindRefs(locp, srcp, srcp->pgno);
for (aprev = locp->vprev; *aprev; ++aprev)
PUTLINK(locp, *aprev, dstp->pgno);
ztail->used += srcp->used;
srcp->used = srcp->recs = 0;
BUFLINK(locp, srcp, 0);
_hxsave(locp, srcp);
_hxalloc(locp, srcp->pgno, 0);
++atail;
}
// Now decide whether to grow or shrink the file.
PAGENO overflows = 0;
for (pg = 1; pg < locp->npages; ++pg) {
if (IS_HEAD(pg)) {
int loops = HX_MAX_CHAIN;
for (pm = pg; (pm = locp->vnext[pm]); ++overflows)
if (!--loops)
LEAVE(locp, HXERR_BAD_FILE);
}
}
PAGENO dpages = locp->dpages + overflows;
PAGENO goodsize = dpages / (1.0 + overload);
DEBUG("%.0f/%.0f=%0.f %.0f %lu/%.2f=%lu => %lu",
fullbytes, fullpages, fullbytes / fullpages,
totbytes, dpages, overload + 1, goodsize, _hxd2f(goodsize));
// "+1" for the root page
goodsize = goodsize ? _hxd2f(goodsize) + 1 : 2;
if (locp->npages <= goodsize) {
// Increase dpages.
// Note that _hxgrow always returns an ALLOCATED
// overflow. It would be smarter to clear all the
// map bits in one step at the end, the way
// hxbuild sets all the map bits in one load/save.
PAGENO junk = 0;
while (locp->npages < goodsize) {
_hxgrow(locp, dstp, DATASIZE(hp), &junk);
_hxsave(locp, dstp);
_hxalloc(locp, dstp->pgno, 0);
}
_hxflushfreed(locp, dstp);
LEAVE(locp, HXNOTE);
}
// Build a list of free pgnos
assert(sizeof *afree <= sizeof *locp->vtail);
afree = zfree = (PAGENO *) locp->vtail;
for (pg = pm = 0; pg < locp->npages; pg += HXPGRATE) {
if (!VREF(locp, pg) && !IS_MAP(hp, pg))
*zfree++ = pg;
}
// Since we are decrementing npages BEFORE reading last page,
// set locked such that _hxislocked gives correct answer.
hp->locked |= LOCKED_BEYOND;
// Work backward from end of file, trimming pages.
for (; locp->npages > goodsize; --locp->npages) {
PAGENO srchead = locp->npages - 1;
PAGENO srctail, dsthead, dstneck, dsttail;
int loops = HX_MAX_CHAIN;
// EASIEST CASE: a map or unreferenced oveflow page
if (srchead == zfree[-1] || IS_MAP(hp, srchead)) {
assert(!VREF(locp, srchead) && !IS_HEAD(srchead));
--zfree;
continue;
}
// EASIER CASE: an empty head page
_hxload(locp, srcp, srchead);
if (!srcp->used)
continue;
// Anything from here on might need 2 free pages
if (zfree - afree < 2)
break;
--VREF(locp, srcp->next);
STAIN(srcp);
srcp->pgno = *afree++;
# if 0
// hxshape does not work with MMAP as yet.
if (hp->mmap)
memcpy(&hp->mmap[(off_t) srcp->pgno * hp->pgsize],
srcp->page, hp->pgsize);
# endif
_hxalloc(locp, srcp->pgno, 1);
_hxsetRef(locp, srcp->pgno, srcp->next);
// EASY CASE: an overflow page to relocate:
if (!IS_HEAD(srchead)) {
_hxsave(locp, srcp);
_hxfindRefs(locp, srcp, srchead);
for (aprev = locp->vprev; *aprev; ++aprev)
PUTLINK(locp, *aprev, srcp->pgno);
continue;
}
// HARD CASE: a head page to desplit:
locp->hash = RECHASH(srcp->data);
_hxpoint(locp); // recalc (dpages,head)
dsthead = locp->head;
dstneck = locp->vnext[dsthead];
dsttail = _hxgetRef(locp, dsthead, 0);
srctail = _hxgetRef(locp, srchead, 0);
// Append srchead to dsttail, or insert chain between
// dsthead and vnext[dsthead]. If srctail is shared,
// make a copy of * it first.
if (dsttail == dsthead || VREF(locp, dsttail) == 1) {
dsthead = dsttail;
} else if (srctail == srchead) {
BUFLINK(locp, srcp, dstneck);
} else if (VREF(locp, srctail) == 1) {
PUTLINK(locp, srctail, dstneck);
} else if (srctail == dsttail) {
if (dsttail != dstneck) {
srctail = _hxgetRef(locp, srcp->pgno, srctail);
if (srctail == srcp->pgno) {
BUFLINK(locp, srcp, dstneck);
} else {
PUTLINK(locp, srctail, dstneck);
}
}
} else {
_hxload(locp, oldp, srctail);
if (!oldp->used)
LEAVE(locp, HXERR_BAD_FILE);
_hxfresh(locp, dstp, *afree++);
_hxalloc(locp, dstp->pgno, 1);
_hxshift(locp, locp->head, srchead, oldp, dstp, dstp);
_hxsave(locp, oldp);
// This hack prevents the CHECK in _hxlink from
// aborting when oldp contains a page that the
// next iteration wants to PUTLINK. This ONLY occurs
// in this code (I think).
// TODO: can this happen in (hxfix,hxshape)??
oldp->pgno = -1;
BUFLINK(locp, dstp, dstneck);
_hxsave(locp, dstp);
if (srcp->next == srctail) {
BUFLINK(locp, srcp, dstp->pgno);
} else {
pg = _hxgetRef(locp, srchead, srctail);
PUTLINK(locp, pg, dstp->pgno);
}
}
_hxload(locp, dstp, dsthead);
BUFLINK(locp, dstp, srcp->pgno);
// Cannot early-out on !SHRUNK here (as "hxput" does);
// new chain may have vacancies in two places.
while (1) {
if (!--loops)
LEAVE(locp, HXERR_BAD_FILE);
if (_hxshift(locp, locp->head, srchead, srcp, dstp, dstp)) {
SWAP(srcp, dstp);
} else {
BUFLINK(locp, dstp, srcp->next);
if (!srcp->used != !VREF(locp, srcp->pgno))
LEAVE(locp, HXERR_BAD_FILE);
if (!srcp->used) {
BUFLINK(locp, srcp, 0);
*--afree = srcp->pgno;
_hxalloc(locp, srcp->pgno, 0);
}
}
_hxsave(locp, srcp);
if (!dstp->next)
break;
_hxload(locp, srcp, dstp->next);
}
_hxsave(locp, dstp);
}
// npages was overdecremented by one in loop
_hxresize(locp, locp->npages + 1);
// Zero the freemap for all truncated overflow pages:
pg = _hxmap(hp, locp->npages + HXPGRATE
- locp->npages % HXPGRATE, &bitpos);
_hxload(locp, dstp, pg);
DEBUG2("clear map %lu from bit %d onward", pg, bitpos);
pos = bitpos >> 3;
dstp->data[pos++] &= ~(-1 << (bitpos & 7));
memset(dstp->data + pos, 0, DATASIZE(hp) - pos);
STAIN(dstp);
_hxsave(locp, dstp);
LEAVE(locp, HXOKAY);
}
