Ejemplo n.º 1
0
Archivo: build_t.c Proyecto: rsanaie/hx
int
main(void)
{
    char    input[999];
    char const *envhx = getenv("hx");
    if (!envhx)         die(": $hx not set");

    setvbuf(stdout, 0, _IOLBF, 0);
     
    plan_tests(1 + 4*4);
    
    HXRET   rc = hxbuild(0, 0, 1<<20, 0.0);
    ok(rc == HXERR_BAD_REQUEST, "hxbuild rejects NULL arg: %s", hxerror(rc));

    try("/dev/null", 0);
    try(strcat(strcpy(input, envhx), "/data.tab"), 88172);
    try(strcat(strcat(strcpy(input, "cat "), envhx), "/data.tab"), 88172);
    try(strcat(strcat(strcpy(input, "head -2335 "), envhx), "/data.tab"), 2335);

    return exit_status();
}

static void
try(char const *inpfile, int nrecs)
{
    FILE        *fp = strchr(inpfile, ' ') ? popen(inpfile, "r") : fopen(inpfile, "r");
    ok(fp, "input %s", inpfile);
    if (!fp) die(": unable to open %s:", inpfile);
    setvbuf(fp, NULL, _IOFBF, 16384);

    hxcreate("build_t.hx", 0755, 4096, "", 0);

    HXFILE      *hp = hxopen("build_t.hx", HX_UPDATE);

    double      alpha = tick();
    HXRET       rc = hxbuild(hp, fp, 1<<20, 0.0); //128MB
    double      omega = tick();

    if (strchr(inpfile, ' ')) pclose(fp); else fclose(fp);

    ok(rc == 0, "hxbuild from (%s) in %.3g secs, returns %s", inpfile, omega - alpha, hxerror(rc));

    rc = hxfix(hp,0,0,0,0);
    ok(rc == (HXRET)HX_UPDATE, "hxcheck returns: %s", hxmode(rc));

    HXSTAT      st;
    hxstat(hp, &st);
    ok(st.nrecs == nrecs, "hxbuild loaded %.0f/%d records", st.nrecs, nrecs);
    hxclose(hp);
    unlink("build_t.hx");
}
Ejemplo n.º 2
0
int
main(int argc, char **argv)
{
    int     count = argc > 1 ? atoi(argv[1]) : 10000;

    plan_tests(9);

    ok(sizeof(off_t) == 8, "sizeof(off_t) == %" FSIZE "d", sizeof(off_t));

    // TODO: add tests that actually test hx() on large databases
    int     npages = 1 << 20;
    int     pgsize = 8192;
    int     rc = hxcreate("large_t.hx", 0666, pgsize, NULL, 0);

    ok(rc == HXOKAY, "hxcreate(large_t.hx): %d %s", rc, hxerror(rc));

    HXFILE *hp = hxopen("large_t.hx", HX_UPDATE);

    ok(hp != NULL, "hxopen(large_t.hx): %d %s", errno, errname[errno]);
    int     fd = hxfileno(hp);
    off_t   size = (off_t) pgsize * npages; // 8GB

    ok(!ftruncate(fd, size), "grow to %" FOFF "d: %d %s", size, errno,
       errname[errno]);
    // Manually initialize map pages.
    // A map page must indicate that it itself (i.e. bit 0 in the map) is
    // allocated, else it will eventually be allocated and overwritten as
    // an overflow page.
    // 32:8(bits/byte)*4(pgrate) pgsize:bytes/page  6:(page header overhead)
    int     pg, last = _hxmap(hp, npages, &pg); // "pg" is just junk here.
    char    map = 1;

    for (pg = 0; (pg = NEXT_MAP(hp, pg)) <= last;) {
        lseek(fd, (off_t) pg * pgsize + sizeof(HXPAGE), 0);
        write(fd, &map, 1);
    }

    struct stat sb;

    ok(!fstat(hxfileno(hp), &sb), "fstat: %d %s", errno, errname[errno]);
    ok(sb.st_size == size, "size is 0x%" FOFF "x", sb.st_size);

    // 10000 records is enough to ensure that at least SOME records end up
    // above the 4GB mark.
    int     i, tally = 0;
    char    rec[11] = { };
    for (i = rc = 0; i < count; ++i) {
        sprintf(rec, "%08d", i);
        rc = hxput(hp, rec, 10);
        if (rc != 0)
            break;
        rc = hxget(hp, rec, 10);
        if (rc != 10) {
            fprintf(stderr, "hxget(%s) failed: %d\n", rec, rc);
            break;
        }
    }
    ok(i == count, "inserted and retrieved %d/%d records: %s", i, count,
       hxerror(rc));

    while (0 < (rc = hxnext(hp, rec, 11)))
        ++tally;
    ok(tally == count, "hxnext retrieved %d/%d records: %s", tally, count,
       hxerror(rc));

    rc = hxfix(hp, 0, 0, 0, 0);
    ok(rc == (HXRET) HX_UPDATE, "hxcheck: large_t.hx is ready for %s access",
       hxmode(rc));

    hxclose(hp);

    return exit_status();
}
Ejemplo n.º 3
0
int
main(int argc, char **argv)
{
    if (argc == 1)
        return fputs("Usage: hxample key ...\n", stderr);

    // Pagesize 4096 is realistic:
    int     rc = hxcreate("hxample.hx", /*perms */ 0664, /*pgsize */ 64,
                          /*rectype name */ "", /*strlen("") */ 0);

    printf("  hxcreate hxample.hx: %s\n", hxerror(rc));

    if (access("hx_.so", R_OK | X_OK))
        return fputs("'hx_.so' must be in this directory\n", stderr);
    char    ld_library_path[] = "LD_LIBRARY_PATH=.";

    putenv(ld_library_path);

    // HX_UPDATE + HX_MMAP for speed ...
    HXFILE *hp = hxopen("hxample.hx", HX_UPDATE);

    if (!hp)
        return fputs("Unable to open hxample.hx!?\n", stderr);

    char    buf[999], *val;
    int     i, len;

    // Insert (key,index)
    puts("# Insert keys with numeric values");
    for (i = 1; i < argc; ++i) {
        len = strlen(argv[i]);
        strcpy(buf, argv[i]);
        sprintf(val = buf + len + 1, "%08d", i);
        rc = hxput(hp, buf, len + 10);
        printf("  hxput(%s: %s) returned %d %s\n",
               buf, val, rc,
               rc < 0 ? hxerror(rc) : rc > 0 ? "replacement" : "");
    }

    puts("# Retrieve by key in reverse order");
    while (--i) {
        strcpy(buf, argv[i]);
        len = hxget(hp, buf, sizeof buf);
        val = buf + strlen(buf) + 1;    // Assumes the hxget succeeded.
        printf("  hxget(%s) returned %d\t%s(%s: %s)\n", argv[i],
               len, len < 0 ? hxerror(len) : "", buf, val);
    }

    puts("# Retrieve records in a hxnext loop");
    while ((len = hxnext(hp, buf, sizeof buf)) > 0) {
        val = buf + strlen(buf) + 1;
        printf("  hxnext returned %d\t%s(%s,%s)\n",
               len, len < 0 ? hxerror(len) : "", buf, val);
    }

    puts("# Append '-<key>' to each value");
    while (++i < argc) {
        strcpy(buf, argv[i]);
        // hxhold locks the record; superfluous here.
        len = hxhold(hp, buf, sizeof buf);
        printf("  hxhold returned %d\n", len);
        val = buf + strlen(buf) + 1;
        strcat(strcat(val, "-"), buf);
        len = hxput(hp, buf, len + 1 + strlen(buf));
        printf("  hxput(%s:%s) returned %d %s\n",
               buf, val, len, len < 0 ? hxerror(len) : "");
    }
    hxdebug = 3;
    puts("# Retrieve records after append, with hxdebug output");
    while ((len = hxnext(hp, buf, sizeof buf)) > 0) {
        val = buf + strlen(buf) + 1;
        printf("  hxnext returned %d\t%s(%s,%s)\n",
               len, len < 0 ? hxerror(len) : "", buf, val);
    }
    hxclose(hp);

    puts("# Now try some commands:\n"
         "  $ chx help\n"
         "  $ chx info hxample.hx\n"
         "  $ chx save hxample.hx\n" "  $ chx -v check hxample.hx\n");

    return 0;
}
Ejemplo n.º 4
0
Archivo: perf_x.c Proyecto: rsanaie/hx
int
main(int argc, char **argv)
{
    char const *hx = getenv("hx");
    if (!hx) hx = ".";
    int         mapmode = HX_MMAP;
    char        inpdflt[strlen(hx) + sizeof "/data.tab"];
    char const *inpfile = inpdflt;
    int         memsize = 0;
    strcat(strcpy(inpdflt, hx), "/data.tab");

    if (argc == 1) usage("[-d] inpfile [membits | 0]\n"
                         "   -d: use disk io, not mmap\n"
                         "   membits: size of RAM for hxbuild (default:20)\n"
                         "       membits=0 means use the existing perf_x.hx");
    if (!strcmp(argv[1], "-d"))
        mapmode = 0, ++argv, --argc;

    switch (argc) {
    case 3: memsize = atoi(argv[2]);
    case 2: inpfile = argv[1];
    case 1: break;
    }

    // Ensure that dirname(argv[0]) is in LD_LIBRARY_PATH:
    char *dir = strrchr(argv[0], '/'), empty[] = "";
    char const *llp = getenv("LD_LIBRARY_PATH");
    dir = dir ? (*dir = 0, argv[0]) : empty;
    if (!llp) llp = "";
    char llpath[strlen(dir) + strlen(llp) + 2];
    setenv("LD_LIBRARY_PATH", strcat(strcat(strcpy(llpath, dir), ":"), llp), /*OVERRIDE*/1);

    HXFILE  *hp;
    HXSTAT  info;
    HXRET   rc;

    FILE        *fp = strcmp(inpfile, "-") ? fopen(inpfile, "r") : stdin;
    if (!fp) die(": cannot read %s:", inpfile);
    setvbuf(fp, NULL, _IOFBF, 65536);

    if (memsize) {
        memsize = 1 << memsize;
        hxcreate("perf_x.hx", 0644, 4096, 0, 0);
        hp = hxopen("perf_x.hx", HX_UPDATE);
        rc = hxbuild(hp, fp, memsize, 0.0);
        if (rc < 0) die("hxbuild(%d): %s", memsize, hxerror(rc));
        hxclose(hp);
    }

    if (hxdebug) system("echo;echo built; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");
    hp = hxopen("perf_x.hx", HX_UPDATE | mapmode);
    if (!hp) die("cannot open perf_x.hx%s:", mapmode ? " with MMAP" : ""); 

    char    buf[4096], rec[4096];
    int     len;
    double t0 = tick();

    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        buf[strlen(buf) - 1] = 0;
        hx_load(hp, rec, sizeof rec, buf);
        hxget(hp, rec, sizeof(rec));
    }
    t0 = tick() - t0;

    double  t1 = tick();
    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        buf[strlen(buf) - 1] = 0;
        buf[0] ^= 0x55;
        hx_load(hp, rec, sizeof rec, buf);
        hxget(hp, rec, sizeof(rec));
    }
    t1 = tick() - t1;

    double  t2 = tick();
    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        buf[strlen(buf) - 1] = 0;
        strcat(buf, "hello, world");
        len = hx_load(hp, rec, sizeof rec, buf);
        hxput(hp, rec, len);
    }
    t2 = tick() - t2;
    if (hxdebug) system("echo;echo put+12; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");

    double  t3 = tick();
    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        buf[strlen(buf) - 1] = 0;
        len = hx_load(hp, rec, sizeof rec, buf);
        hxput(hp, rec, len);
    }
    t3 = tick() - t3;
    if (hxdebug) system("echo;echo put-0; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");

    double  t4 = tick();
    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        strcpy(buf + strlen(buf) - 1, "---------1---------2---------3---------4---------5---------6---------7---------8---------9---------0\n");
        len = hx_load(hp, rec, sizeof rec, buf);
        hxput(hp, rec, len);
    }
    t4 = tick() - t4;
    if (hxdebug) system("echo;echo put+100; chx info perf_x.hx; chx stat perf_x.hx; chx -vdd check perf_x.hx");

    double  t5 = tick();
    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        buf[strlen(buf) - 1] = 0;
        hx_load(hp, rec, sizeof rec, buf);
        hxget(hp, rec, sizeof(rec));
    }
    t5 = tick() - t5;

    double  t6 = tick();
    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        buf[strlen(buf) - 1] = 0;
        buf[0] ^= 0x55;
        hx_load(hp, rec, sizeof rec, buf);
        hxget(hp, rec, sizeof(rec));
    }
    t6 = tick() - t6;

    double  t7 = tick();
    rewind(fp);
    while (fgets(buf, sizeof(buf), fp)) {
        buf[strlen(buf) - 1] = 0;
        buf[0] ^= 0x55;
        hx_load(hp, rec, sizeof rec, buf);
        hxput(hp, rec, sizeof(rec));
    }
    t7 = tick() - t7;

    hxstat(hp, &info);
    fprintf(stderr, "nrecs: %g build: %.2fM rec/sec\n(usec:)\n"
                    "\tget-y\t%.2f\n\tget-n\t%.2f\n"
                    "\tput+12\t%.2f\n\tput+0\t%.2f\n\tput+100\t%.2f\n"
                    "\tget-y\t%.2f\n\tget-n\t%.2f\n\tput-xx\t%.2f\n",
            info.nrecs, info.nrecs/1E6/t0, t0*1E6/info.nrecs,
                        t1*1E6/info.nrecs, t2*1E6/info.nrecs,
                        t3*1E6/info.nrecs, t4*1E6/info.nrecs,
                        t5*1E6/info.nrecs, t6*1E6/info.nrecs,
                        t7*1E6/info.nrecs);

    return  0;
}
Ejemplo n.º 5
0
Archivo: chx.c Proyecto: mischasan/hx
//--------------|---------------------------------------------
int
main(int argc, char **argv)
{
    int     ret = 0;
    int     opt;
    int     timed = 0;
    char    cmd[10240];

    while (0 < (opt = getopt(argc, argv, "?c:dmp:s:tv-"))) {
        switch (opt) {

        case '?':
            help();
            break;
        case 'c':
            hxcrash = atoi(optarg);
            break;
        case 'd':
            ++hxdebug;
            break;
        case 'm':
            mmode |= HX_MMAP;
            break;
        case 's':
            mmode |= HX_FSYNC;
            break;
        case 't':
            timed++;
            break;
        case 'v':
            ++verbose;
            break;
        }
    }

    argc -= optind;
    argv += optind;

    setvbuf(stdout, NULL, _IOLBF, 0);
    setvbuf(stderr, NULL, _IOLBF, 0);
    errno = 0;                  //setvbuf sets errno !?

    int     size = 0;
    HXMODE  mode;
    HXFILE *hp = NULL;
    FILE   *fp = NULL;

    if (!*argv)
        die("See 'chx help' for usage");
    if (*argv[0] == '?' || !strcmp(argv[0], "help"))
        help();

    double  tstart = tick();

    if (hxdebug)
        hxtime = tstart;
    if (!strcmp(argv[0], "build")) {

        hp = do_hxopen("build", argv[1], HX_UPDATE);
        fp = do_fopen("build", argv[2], "r");
        int     memsize = argc > 3 ? atoi(argv[3]) : 1;
        int     inpsize = argc > 4 ? atoi(argv[4]) : 0;

        memsize <<= 20;
        is_hxret("build", hxbuild(hp, fp, memsize, inpsize));

    } else if (!strcmp(argv[0], "check")) {
        char   *udata = argc > 3 ? argv[3] : NULL;

        if (argc < 2)
            die("%s: requires filename [pgsize [udata]]", argv[0]);
        if (argc > 2 && !sscanf(argv[2], "%d", &size))
            die("%s: invalid pgsize", argv[2]);

        hp = do_hxopen("check", argv[1], HX_CHECK);
        mode = hxfix(hp, NULL, size, udata, udata ? strlen(udata) : 0);
        if (verbose || mode != HX_UPDATE)
            printf("%s %s\n", hxmode(mode), errno ? strerror(errno) : "");
        ret = mode != HX_UPDATE;

    } else if (!strcmp(argv[0], "create")) {

        if (argc < 3 || !sscanf(argv[2], "%d", &size))
            die("create: requires filename, pgsize, type");

        char const *type = argv[3] ? argv[3] : "";

        is_hxret("create", hxcreate(argv[1], 0644, size, type, strlen(type)));

        hp = do_hxopen("create", argv[1], HX_RECOVER);

    } else if (!strcmp(argv[0], "del")) {

        hp = do_hxopen("del", argv[1], HX_UPDATE);
        fp = do_fopen("del", argv[2], "r");
        del(hp, fp);

    } else if (!strcmp(argv[0], "dump")) {

        hp = do_hxopen("dump", argv[1], HX_READ);
        dump(hp, stdout);

    } else if (!strcmp(argv[0], "fix") || !strcmp(argv[0], "repair")) {
        char   *udata = argc > 3 ? argv[3] : NULL;

        if (argc < 2)
            die("%s: requires filename [pgsize [udata]]", argv[0]);

        if (argv[2] && !sscanf(argv[2], "%d", &size))
            die("%s: invalid pgsize", argv[2]);

        hp = do_hxopen("repair", argv[1], HX_UPDATE);
        fp = tmpfile();
        mode = hxfix(hp, fp, size, udata, udata ? strlen(udata) : 0);
        if (verbose || mode != HX_UPDATE)
            printf("%s %s\n", hxmode(mode), errno ? strerror(errno) : "");
        ret = mode != HX_UPDATE;

    } else if (!strcmp(argv[0], "hdrs")) {

        hp = do_hxopen("hdrs", argv[1], HX_READ);
        fp = do_fopen("hdrs", argv[2], "w");
        hdrs(hp);

    } else if (!strcmp(argv[0], "info")) {

        hp = do_hxopen("info", argv[1], HX_READ);
        info(hp);

    } else if (!strcmp(argv[0], "load")) {

        hp = do_hxopen("load", argv[1], HX_UPDATE);
        fp = do_fopen("hdrs", argv[2], "r");
        do_load(hp, fp);

    } else if (!strcmp(argv[0], "lock")) {

        hp = do_hxopen("lock", argv[1], HX_READ);
        do_lock(hp);

    } else if (!strcmp(argv[0], "maps")) {

        hp = do_hxopen("maps", argv[1], HX_READ);
        is_hxret("maps", maps(hp));

    } else if (!strcmp(argv[0], "pack")) {

        hp = do_hxopen("load", argv[1], HX_UPDATE);
        is_hxret("pack", hxpack(hp));

    } else if (!strcmp(argv[0], "save")) {

        hp = do_hxopen("save", argv[1], HX_READ);
        fp = do_fopen("save", argv[2], "w");
        do_save(hp, fp);

    } else if (!strcmp(argv[0], "shape")) {
        double  density;

        if (argc != 3 || !sscanf(argv[2], "%lf", &density))
            die("%s: requires density arg (0 to 1.0)", argv[2]);

        hp = do_hxopen("shape", argv[1], HX_UPDATE);
        is_hxret("shape", hxshape(hp, density));

    } else if (!strcmp(argv[0], "stat")) {

        hp = do_hxopen("stat", argv[1], HX_READ);
        stats(hp);

    } else if (!strcmp(argv[0], "types")) {
        // for each dir in LD_LIBRARY_PATH (and "" -> ".") then /lib then /usr/lib,
        //  find all files of the form "<dir>/hx_<rectype>.so"
        //      and build a file of unique rectypes.
        // Then call hxlib which returns an exact path and a bitmask of DIFF/LOAD/TEST
        //  "type diff-load-test path"
        // If the path matches, print the entry.

        types(getenv("LD_LIBRARY_PATH"));
        types("lib:/usr/lib");
    } else {

        die("%s: unknown command. See 'chx help'", cmd);
    }

    if (fp)
        fclose(fp);
    if (hp)
        hxclose(hp);
    if (timed)
        fprintf(stderr, "# chx %s: %.3f secs\n", *argv, tick() - tstart);

    return ret;
}