static void
do_load(HXFILE * hp, FILE * inp)
{
HXRET hxret = 0;
int reclen, recsize = hxmaxrec(hp);
int bufsize = 2 * recsize;
char rec[recsize], buf[bufsize];
int lineno = 0, more, added = 0;
if (verbose == 1)
setvbuf(stderr, NULL, _IONBF, 0);
while ((more = ! !fgets(buf, bufsize, inp))) {
++lineno;
char *cp = buf + strlen(buf);
if (cp > buf && cp[-1] == '\n')
*--cp = 0;
if (cp == buf)
continue;
reclen = hx_load(hp, rec, recsize, buf);
if (reclen <= 0) {
fprintf(stderr, "# load: invalid %s: %s\n",
reclen == HXERR_BAD_REQUEST ? "request" : "input", buf);
continue;
}
hxret = hxput(hp, rec, reclen);
if (verbose > 1)
fprintf(stderr, "# load(%s): %s\n", buf, hxerror(hxret));
else if (verbose && lineno % 1000 == 0)
putc('.', stderr);
if (hxret < 0)
break;
if (hxret == HXNOTE)
++added;
}
if (verbose == 1 && lineno > 999)
putc('\n', stderr);
if (more)
fprintf(stderr, "load: error at input line %d (%s)\n",
lineno, hxerror(hxret));
if (verbose)
fprintf(stderr, "load: in: %d added: %d\n", lineno, added);
}
int
main(int argc, char **argv)
{
int count = argc > 1 ? atoi(argv[1]) : 10000;
plan_tests(9);
ok(sizeof(off_t) == 8, "sizeof(off_t) == %" FSIZE "d", sizeof(off_t));
// TODO: add tests that actually test hx() on large databases
int npages = 1 << 20;
int pgsize = 8192;
int rc = hxcreate("large_t.hx", 0666, pgsize, NULL, 0);
ok(rc == HXOKAY, "hxcreate(large_t.hx): %d %s", rc, hxerror(rc));
HXFILE *hp = hxopen("large_t.hx", HX_UPDATE);
ok(hp != NULL, "hxopen(large_t.hx): %d %s", errno, errname[errno]);
int fd = hxfileno(hp);
off_t size = (off_t) pgsize * npages; // 8GB
ok(!ftruncate(fd, size), "grow to %" FOFF "d: %d %s", size, errno,
errname[errno]);
// Manually initialize map pages.
// A map page must indicate that it itself (i.e. bit 0 in the map) is
// allocated, else it will eventually be allocated and overwritten as
// an overflow page.
// 32:8(bits/byte)*4(pgrate) pgsize:bytes/page 6:(page header overhead)
int pg, last = _hxmap(hp, npages, &pg); // "pg" is just junk here.
char map = 1;
for (pg = 0; (pg = NEXT_MAP(hp, pg)) <= last;) {
lseek(fd, (off_t) pg * pgsize + sizeof(HXPAGE), 0);
write(fd, &map, 1);
}
struct stat sb;
ok(!fstat(hxfileno(hp), &sb), "fstat: %d %s", errno, errname[errno]);
ok(sb.st_size == size, "size is 0x%" FOFF "x", sb.st_size);
// 10000 records is enough to ensure that at least SOME records end up
// above the 4GB mark.
int i, tally = 0;
char rec[11] = { };
for (i = rc = 0; i < count; ++i) {
sprintf(rec, "%08d", i);
rc = hxput(hp, rec, 10);
if (rc != 0)
break;
rc = hxget(hp, rec, 10);
if (rc != 10) {
fprintf(stderr, "hxget(%s) failed: %d\n", rec, rc);
break;
}
}
ok(i == count, "inserted and retrieved %d/%d records: %s", i, count,
hxerror(rc));
while (0 < (rc = hxnext(hp, rec, 11)))
++tally;
ok(tally == count, "hxnext retrieved %d/%d records: %s", tally, count,
hxerror(rc));
rc = hxfix(hp, 0, 0, 0, 0);
ok(rc == (HXRET) HX_UPDATE, "hxcheck: large_t.hx is ready for %s access",
hxmode(rc));
hxclose(hp);
return exit_status();
}
int
main(int argc, char **argv)
{
char const *hx = getenv("hx");
if (!hx) hx = ".";
int mapmode = HX_MMAP;
char inpdflt[strlen(hx) + sizeof "/data.tab"];
char const *inpfile = inpdflt;
int memsize = 0;
strcat(strcpy(inpdflt, hx), "/data.tab");
if (argc == 1) usage("[-d] inpfile [membits | 0]\n"
" -d: use disk io, not mmap\n"
" membits: size of RAM for hxbuild (default:20)\n"
" membits=0 means use the existing perf_x.hx");
if (!strcmp(argv[1], "-d"))
mapmode = 0, ++argv, --argc;
switch (argc) {
case 3: memsize = atoi(argv[2]);
case 2: inpfile = argv[1];
case 1: break;
}
// Ensure that dirname(argv[0]) is in LD_LIBRARY_PATH:
char *dir = strrchr(argv[0], '/'), empty[] = "";
char const *llp = getenv("LD_LIBRARY_PATH");
dir = dir ? (*dir = 0, argv[0]) : empty;
if (!llp) llp = "";
char llpath[strlen(dir) + strlen(llp) + 2];
setenv("LD_LIBRARY_PATH", strcat(strcat(strcpy(llpath, dir), ":"), llp), /*OVERRIDE*/1);
HXFILE *hp;
HXSTAT info;
HXRET rc;
FILE *fp = strcmp(inpfile, "-") ? fopen(inpfile, "r") : stdin;
if (!fp) die(": cannot read %s:", inpfile);
setvbuf(fp, NULL, _IOFBF, 65536);
if (memsize) {
memsize = 1 << memsize;
hxcreate("perf_x.hx", 0644, 4096, 0, 0);
hp = hxopen("perf_x.hx", HX_UPDATE);
rc = hxbuild(hp, fp, memsize, 0.0);
if (rc < 0) die("hxbuild(%d): %s", memsize, hxerror(rc));
hxclose(hp);
}
if (hxdebug) system("echo;echo built; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");
hp = hxopen("perf_x.hx", HX_UPDATE | mapmode);
if (!hp) die("cannot open perf_x.hx%s:", mapmode ? " with MMAP" : "");
char buf[4096], rec[4096];
int len;
double t0 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
buf[strlen(buf) - 1] = 0;
hx_load(hp, rec, sizeof rec, buf);
hxget(hp, rec, sizeof(rec));
}
t0 = tick() - t0;
double t1 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
buf[strlen(buf) - 1] = 0;
buf[0] ^= 0x55;
hx_load(hp, rec, sizeof rec, buf);
hxget(hp, rec, sizeof(rec));
}
t1 = tick() - t1;
double t2 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
buf[strlen(buf) - 1] = 0;
strcat(buf, "hello, world");
len = hx_load(hp, rec, sizeof rec, buf);
hxput(hp, rec, len);
}
t2 = tick() - t2;
if (hxdebug) system("echo;echo put+12; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");
double t3 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
buf[strlen(buf) - 1] = 0;
len = hx_load(hp, rec, sizeof rec, buf);
hxput(hp, rec, len);
}
t3 = tick() - t3;
if (hxdebug) system("echo;echo put-0; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");
double t4 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
strcpy(buf + strlen(buf) - 1, "---------1---------2---------3---------4---------5---------6---------7---------8---------9---------0\n");
len = hx_load(hp, rec, sizeof rec, buf);
hxput(hp, rec, len);
}
t4 = tick() - t4;
if (hxdebug) system("echo;echo put+100; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");
double t5 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
buf[strlen(buf) - 1] = 0;
hx_load(hp, rec, sizeof rec, buf);
hxget(hp, rec, sizeof(rec));
}
t5 = tick() - t5;
double t6 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
buf[strlen(buf) - 1] = 0;
buf[0] ^= 0x55;
hx_load(hp, rec, sizeof rec, buf);
hxget(hp, rec, sizeof(rec));
}
t6 = tick() - t6;
double t7 = tick();
rewind(fp);
while (fgets(buf, sizeof(buf), fp)) {
buf[strlen(buf) - 1] = 0;
buf[0] ^= 0x55;
hx_load(hp, rec, sizeof rec, buf);
hxput(hp, rec, sizeof(rec));
}
t7 = tick() - t7;
hxstat(hp, &info);
fprintf(stderr, "nrecs: %g build: %.2fM rec/sec\n(usec:)\n"
"\tget-y\t%.2f\n\tget-n\t%.2f\n"
"\tput+12\t%.2f\n\tput+0\t%.2f\n\tput+100\t%.2f\n"
"\tget-y\t%.2f\n\tget-n\t%.2f\n\tput-xx\t%.2f\n",
info.nrecs, info.nrecs/1E6/t0, t0*1E6/info.nrecs,
t1*1E6/info.nrecs, t2*1E6/info.nrecs,
t3*1E6/info.nrecs, t4*1E6/info.nrecs,
t5*1E6/info.nrecs, t6*1E6/info.nrecs,
t7*1E6/info.nrecs);
return 0;
}
int
main(int argc, char **argv)
{
if (argc == 1)
return fputs("Usage: hxample key ...\n", stderr);
// Pagesize 4096 is realistic:
int rc = hxcreate("hxample.hx", /*perms */ 0664, /*pgsize */ 64,
/*rectype name */ "", /*strlen("") */ 0);
printf(" hxcreate hxample.hx: %s\n", hxerror(rc));
if (access("hx_.so", R_OK | X_OK))
return fputs("'hx_.so' must be in this directory\n", stderr);
char ld_library_path[] = "LD_LIBRARY_PATH=.";
putenv(ld_library_path);
// HX_UPDATE + HX_MMAP for speed ...
HXFILE *hp = hxopen("hxample.hx", HX_UPDATE);
if (!hp)
return fputs("Unable to open hxample.hx!?\n", stderr);
char buf[999], *val;
int i, len;
// Insert (key,index)
puts("# Insert keys with numeric values");
for (i = 1; i < argc; ++i) {
len = strlen(argv[i]);
strcpy(buf, argv[i]);
sprintf(val = buf + len + 1, "%08d", i);
rc = hxput(hp, buf, len + 10);
printf(" hxput(%s: %s) returned %d %s\n",
buf, val, rc,
rc < 0 ? hxerror(rc) : rc > 0 ? "replacement" : "");
}
puts("# Retrieve by key in reverse order");
while (--i) {
strcpy(buf, argv[i]);
len = hxget(hp, buf, sizeof buf);
val = buf + strlen(buf) + 1; // Assumes the hxget succeeded.
printf(" hxget(%s) returned %d\t%s(%s: %s)\n", argv[i],
len, len < 0 ? hxerror(len) : "", buf, val);
}
puts("# Retrieve records in a hxnext loop");
while ((len = hxnext(hp, buf, sizeof buf)) > 0) {
val = buf + strlen(buf) + 1;
printf(" hxnext returned %d\t%s(%s,%s)\n",
len, len < 0 ? hxerror(len) : "", buf, val);
}
puts("# Append '-<key>' to each value");
while (++i < argc) {
strcpy(buf, argv[i]);
// hxhold locks the record; superfluous here.
len = hxhold(hp, buf, sizeof buf);
printf(" hxhold returned %d\n", len);
val = buf + strlen(buf) + 1;
strcat(strcat(val, "-"), buf);
len = hxput(hp, buf, len + 1 + strlen(buf));
printf(" hxput(%s:%s) returned %d %s\n",
buf, val, len, len < 0 ? hxerror(len) : "");
}
hxdebug = 3;
puts("# Retrieve records after append, with hxdebug output");
while ((len = hxnext(hp, buf, sizeof buf)) > 0) {
val = buf + strlen(buf) + 1;
printf(" hxnext returned %d\t%s(%s,%s)\n",
len, len < 0 ? hxerror(len) : "", buf, val);
}
hxclose(hp);
puts("# Now try some commands:\n"
" $ chx help\n"
" $ chx info hxample.hx\n"
" $ chx save hxample.hx\n" " $ chx -v check hxample.hx\n");
return 0;
}