HXRET _hxcheckbuf(HXLOCAL const*locp, HXBUF const*bufp) { HXFILE *hp = locp->file; if (IS_MAP(hp, bufp->pgno)) { if (!(bufp->data[0] & 1)) return bad_map_self; if (bufp->pgno) { // extended map page. if (bufp->next || bufp->used) return bad_map_head; } else { // root page. if (bufp->used >= DATASIZE(hp) || bufp->used != hp->uleng) return bad_map_head; // Todo: check that HXROOT{pgsize,version} are good. } int lastbit, pos; PAGENO lastmap = locp->npages - 1; lastmap = _hxmap(hp, lastmap - lastmap % HXPGRATE, &lastbit); if (bufp->pgno == lastmap) { // All bits beyond lastbit must be zero. BYTE mask = -2 << (lastbit & 7); for (pos = lastbit >> 3; pos < (int)DATASIZE(hp); ++pos, mask = -1) if (bufp->data[pos] & mask) return bad_overmap; } } else { // DATA page.
static HXRET maps(HXFILE * hp) { HXLOCAL loc; PAGENO pg, last; int bitpos; # define locp (&loc) # define bufp (&loc.buf[0]) ENTER(locp, hp, NULL, 1); locp->mode = F_RDLCK; _hxlock(locp, 0, 0); _hxsize(locp); last = locp->npages - 1; last = _hxmap(hp, last - last % HXPGRATE, &bitpos); printf("last: mpg=%u bit=%d\n", (unsigned)last, bitpos); for (pg = 0; pg <= last;) { int i, n; _hxload(locp, bufp, pg); n = pg == last ? (bitpos + 7) >> 3 : (int)DATASIZE(hp); printf("---- pgno=%u next=%u used=%d recs=%u", pg, bufp->next, bufp->used, bufp->recs); for (i = 0; i < n; ++i) { unsigned b = bufp->data[i] & 0xFF; if (!(i & 15)) printf("\n%7u", (unsigned)pg + i * 8 * HXPGRATE); if (i < bufp->used) fputs(" >>", stdout); else printf(" %02X", b); } putchar('\n'); pg += (DATASIZE(hp) - bufp->used) * 8 * HXPGRATE; } LEAVE(locp, 0); # undef locp }
int main(int argc, char **argv) { int count = argc > 1 ? atoi(argv[1]) : 10000; plan_tests(9); ok(sizeof(off_t) == 8, "sizeof(off_t) == %" FSIZE "d", sizeof(off_t)); // TODO: add tests that actually test hx() on large databases int npages = 1 << 20; int pgsize = 8192; int rc = hxcreate("large_t.hx", 0666, pgsize, NULL, 0); ok(rc == HXOKAY, "hxcreate(large_t.hx): %d %s", rc, hxerror(rc)); HXFILE *hp = hxopen("large_t.hx", HX_UPDATE); ok(hp != NULL, "hxopen(large_t.hx): %d %s", errno, errname[errno]); int fd = hxfileno(hp); off_t size = (off_t) pgsize * npages; // 8GB ok(!ftruncate(fd, size), "grow to %" FOFF "d: %d %s", size, errno, errname[errno]); // Manually initialize map pages. // A map page must indicate that it itself (i.e. bit 0 in the map) is // allocated, else it will eventually be allocated and overwritten as // an overflow page. // 32:8(bits/byte)*4(pgrate) pgsize:bytes/page 6:(page header overhead) int pg, last = _hxmap(hp, npages, &pg); // "pg" is just junk here. char map = 1; for (pg = 0; (pg = NEXT_MAP(hp, pg)) <= last;) { lseek(fd, (off_t) pg * pgsize + sizeof(HXPAGE), 0); write(fd, &map, 1); } struct stat sb; ok(!fstat(hxfileno(hp), &sb), "fstat: %d %s", errno, errname[errno]); ok(sb.st_size == size, "size is 0x%" FOFF "x", sb.st_size); // 10000 records is enough to ensure that at least SOME records end up // above the 4GB mark. int i, tally = 0; char rec[11] = { }; for (i = rc = 0; i < count; ++i) { sprintf(rec, "%08d", i); rc = hxput(hp, rec, 10); if (rc != 0) break; rc = hxget(hp, rec, 10); if (rc != 10) { fprintf(stderr, "hxget(%s) failed: %d\n", rec, rc); break; } } ok(i == count, "inserted and retrieved %d/%d records: %s", i, count, hxerror(rc)); while (0 < (rc = hxnext(hp, rec, 11))) ++tally; ok(tally == count, "hxnext retrieved %d/%d records: %s", tally, count, hxerror(rc)); rc = hxfix(hp, 0, 0, 0, 0); ok(rc == (HXRET) HX_UPDATE, "hxcheck: large_t.hx is ready for %s access", hxmode(rc)); hxclose(hp); return exit_status(); }
//--------------|--------------------------------------------- HXRET hxshape(HXFILE * hp, double overload) { HXLOCAL loc, *locp = &loc; HXBUF *srcp, *dstp, *oldp; int pos, bitpos; PGINFO *atail, *ztail; PAGENO pg, pm, *aprev, *afree, *zfree; double totbytes = 0, fullbytes = 0, fullpages = 0; if (!hp || hp->buffer.pgno || hp->mode & HX_MMAP || !(hp->mode & HX_UPDATE)) return HXERR_BAD_REQUEST; ENTER(locp, hp, NULL, 3); _hxlock(locp, 0, 0); _hxsize(locp); srcp = &locp->buf[0]; dstp = &locp->buf[1]; oldp = &locp->buf[2]; _hxinitRefs(locp); // Populate vnext,vrefs,vtail for tail-merging: ztail = calloc(locp->npages / HXPGRATE + 1, sizeof(PGINFO)); locp->vtail = ztail; for (pg = 1; pg < locp->npages; ++pg) { PGINFO x = _hxpginfo(locp, pg); _hxsetRef(locp, pg, x.pgno); totbytes += x.used; if (x.pgno) // i.e. page.next != 0 ++fullpages, fullbytes += x.used; else if (x.used && !IS_HEAD(pg)) x.pgno = pg, *ztail++ = x; } // Sort vtail by (used), so that smallest+largest (used) // counts can be matched up; simple greedy-fill algorithm. qsort(locp->vtail, ztail - locp->vtail, sizeof *locp->vtail, (cmpfn_t) cmpused); // Combine tail pages where possible: for (atail = locp->vtail, --ztail; atail < ztail;) { if (!_FITS(hp, atail->used, atail->recs, ztail->used, ztail->recs)) { --ztail; continue; } // Merge is always from [atail] to [ztail], to maintain // ([ztail].used >= [ztail-1].used). if (atail->pgno < ztail->pgno) { PGINFO tmp = *atail; *atail = *ztail; *ztail = tmp; } _hxload(locp, srcp, atail->pgno); _hxload(locp, dstp, ztail->pgno); _hxappend(dstp, srcp->data, srcp->used); dstp->recs += srcp->recs; _hxsave(locp, dstp); _hxfindRefs(locp, srcp, srcp->pgno); for (aprev = locp->vprev; *aprev; ++aprev) PUTLINK(locp, *aprev, dstp->pgno); ztail->used += srcp->used; srcp->used = srcp->recs = 0; BUFLINK(locp, srcp, 0); _hxsave(locp, srcp); _hxalloc(locp, srcp->pgno, 0); ++atail; } // Now decide whether to grow or shrink the file. PAGENO overflows = 0; for (pg = 1; pg < locp->npages; ++pg) { if (IS_HEAD(pg)) { int loops = HX_MAX_CHAIN; for (pm = pg; (pm = locp->vnext[pm]); ++overflows) if (!--loops) LEAVE(locp, HXERR_BAD_FILE); } } PAGENO dpages = locp->dpages + overflows; PAGENO goodsize = dpages / (1.0 + overload); DEBUG("%.0f/%.0f=%0.f %.0f %lu/%.2f=%lu => %lu", fullbytes, fullpages, fullbytes / fullpages, totbytes, dpages, overload + 1, goodsize, _hxd2f(goodsize)); // "+1" for the root page goodsize = goodsize ? _hxd2f(goodsize) + 1 : 2; if (locp->npages <= goodsize) { // Increase dpages. // Note that _hxgrow always returns an ALLOCATED // overflow. It would be smarter to clear all the // map bits in one step at the end, the way // hxbuild sets all the map bits in one load/save. PAGENO junk = 0; while (locp->npages < goodsize) { _hxgrow(locp, dstp, DATASIZE(hp), &junk); _hxsave(locp, dstp); _hxalloc(locp, dstp->pgno, 0); } _hxflushfreed(locp, dstp); LEAVE(locp, HXNOTE); } // Build a list of free pgnos assert(sizeof *afree <= sizeof *locp->vtail); afree = zfree = (PAGENO *) locp->vtail; for (pg = pm = 0; pg < locp->npages; pg += HXPGRATE) { if (!VREF(locp, pg) && !IS_MAP(hp, pg)) *zfree++ = pg; } // Since we are decrementing npages BEFORE reading last page, // set locked such that _hxislocked gives correct answer. hp->locked |= LOCKED_BEYOND; // Work backward from end of file, trimming pages. for (; locp->npages > goodsize; --locp->npages) { PAGENO srchead = locp->npages - 1; PAGENO srctail, dsthead, dstneck, dsttail; int loops = HX_MAX_CHAIN; // EASIEST CASE: a map or unreferenced oveflow page if (srchead == zfree[-1] || IS_MAP(hp, srchead)) { assert(!VREF(locp, srchead) && !IS_HEAD(srchead)); --zfree; continue; } // EASIER CASE: an empty head page _hxload(locp, srcp, srchead); if (!srcp->used) continue; // Anything from here on might need 2 free pages if (zfree - afree < 2) break; --VREF(locp, srcp->next); STAIN(srcp); srcp->pgno = *afree++; # if 0 // hxshape does not work with MMAP as yet. if (hp->mmap) memcpy(&hp->mmap[(off_t) srcp->pgno * hp->pgsize], srcp->page, hp->pgsize); # endif _hxalloc(locp, srcp->pgno, 1); _hxsetRef(locp, srcp->pgno, srcp->next); // EASY CASE: an overflow page to relocate: if (!IS_HEAD(srchead)) { _hxsave(locp, srcp); _hxfindRefs(locp, srcp, srchead); for (aprev = locp->vprev; *aprev; ++aprev) PUTLINK(locp, *aprev, srcp->pgno); continue; } // HARD CASE: a head page to desplit: locp->hash = RECHASH(srcp->data); _hxpoint(locp); // recalc (dpages,head) dsthead = locp->head; dstneck = locp->vnext[dsthead]; dsttail = _hxgetRef(locp, dsthead, 0); srctail = _hxgetRef(locp, srchead, 0); // Append srchead to dsttail, or insert chain between // dsthead and vnext[dsthead]. If srctail is shared, // make a copy of * it first. if (dsttail == dsthead || VREF(locp, dsttail) == 1) { dsthead = dsttail; } else if (srctail == srchead) { BUFLINK(locp, srcp, dstneck); } else if (VREF(locp, srctail) == 1) { PUTLINK(locp, srctail, dstneck); } else if (srctail == dsttail) { if (dsttail != dstneck) { srctail = _hxgetRef(locp, srcp->pgno, srctail); if (srctail == srcp->pgno) { BUFLINK(locp, srcp, dstneck); } else { PUTLINK(locp, srctail, dstneck); } } } else { _hxload(locp, oldp, srctail); if (!oldp->used) LEAVE(locp, HXERR_BAD_FILE); _hxfresh(locp, dstp, *afree++); _hxalloc(locp, dstp->pgno, 1); _hxshift(locp, locp->head, srchead, oldp, dstp, dstp); _hxsave(locp, oldp); // This hack prevents the CHECK in _hxlink from // aborting when oldp contains a page that the // next iteration wants to PUTLINK. This ONLY occurs // in this code (I think). // TODO: can this happen in (hxfix,hxshape)?? oldp->pgno = -1; BUFLINK(locp, dstp, dstneck); _hxsave(locp, dstp); if (srcp->next == srctail) { BUFLINK(locp, srcp, dstp->pgno); } else { pg = _hxgetRef(locp, srchead, srctail); PUTLINK(locp, pg, dstp->pgno); } } _hxload(locp, dstp, dsthead); BUFLINK(locp, dstp, srcp->pgno); // Cannot early-out on !SHRUNK here (as "hxput" does); // new chain may have vacancies in two places. while (1) { if (!--loops) LEAVE(locp, HXERR_BAD_FILE); if (_hxshift(locp, locp->head, srchead, srcp, dstp, dstp)) { SWAP(srcp, dstp); } else { BUFLINK(locp, dstp, srcp->next); if (!srcp->used != !VREF(locp, srcp->pgno)) LEAVE(locp, HXERR_BAD_FILE); if (!srcp->used) { BUFLINK(locp, srcp, 0); *--afree = srcp->pgno; _hxalloc(locp, srcp->pgno, 0); } } _hxsave(locp, srcp); if (!dstp->next) break; _hxload(locp, srcp, dstp->next); } _hxsave(locp, dstp); } // npages was overdecremented by one in loop _hxresize(locp, locp->npages + 1); // Zero the freemap for all truncated overflow pages: pg = _hxmap(hp, locp->npages + HXPGRATE - locp->npages % HXPGRATE, &bitpos); _hxload(locp, dstp, pg); DEBUG2("clear map %lu from bit %d onward", pg, bitpos); pos = bitpos >> 3; dstp->data[pos++] &= ~(-1 << (bitpos & 7)); memset(dstp->data + pos, 0, DATASIZE(hp) - pos); STAIN(dstp); _hxsave(locp, dstp); LEAVE(locp, HXOKAY); }