Exemple #1
0
static int process_file(const char *file, int *total)
{
    Db *db;
    couchstore_error_t errcode;
    int count = 0;

    errcode = couchstore_open_db(file, COUCHSTORE_OPEN_FLAG_RDONLY, &db);
    if (errcode != COUCHSTORE_SUCCESS) {
        fprintf(stderr, "Failed to open \"%s\": %s\n",
                file, couchstore_strerror(errcode));
        return -1;
    } else {
        fprintf(stderr, "Dumping \"%s\":\n", file);
    }

    switch (mode) {
        case DumpBySequence:
            if (dumpTree) {
                errcode = couchstore_walk_seq_tree(db, 0, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                                   visit_node, &count);
            } else {
                errcode = couchstore_changes_since(db, 0, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                                   foldprint, &count);
            }
            break;
        case DumpByID:
            if (dumpTree) {
                errcode = couchstore_walk_id_tree(db, NULL, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                                  visit_node, &count);
            } else if (oneKey) {
                DocInfo* info;
                errcode = couchstore_docinfo_by_id(db, dumpKey.buf, dumpKey.size, &info);
                if (errcode == COUCHSTORE_SUCCESS) {
                    foldprint(db, info, &count);
                    couchstore_free_docinfo(info);
                }
            } else {
                errcode = couchstore_all_docs(db, NULL, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                              foldprint, &count);
            }
            break;
        case DumpLocals:
            errcode = couchstore_print_local_docs(db, &count);
            break;
    }
    (void)couchstore_close_db(db);

    if (errcode < 0) {
        fprintf(stderr, "Failed to dump database \"%s\": %s\n",
                file, couchstore_strerror(errcode));
        return -1;
    }

    *total += count;
    return 0;
}
Exemple #2
0
LIBCOUCHSTORE_API
couchstore_error_t couchstore_open_db_ex(const char *filename,
        couchstore_open_flags flags,
        const couch_file_ops *ops,
        Db **pDb)
{
    couchstore_error_t errcode = COUCHSTORE_SUCCESS;
    Db *db;
    int openflags;

    /* Sanity check input parameters */
    if ((flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
            (flags & COUCHSTORE_OPEN_FLAG_CREATE)) {
        return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
    }

    if ((db = calloc(1, sizeof(Db))) == NULL) {
        return COUCHSTORE_ERROR_ALLOC_FAIL;
    }

    if (flags & COUCHSTORE_OPEN_FLAG_RDONLY) {
        openflags = O_RDONLY;
    } else {
        openflags = O_RDWR;
    }

    if (flags & COUCHSTORE_OPEN_FLAG_CREATE) {
        openflags |= O_CREAT;
    }

    error_pass(tree_file_open(&db->file, filename, openflags, ops));

    if ((db->file.pos = db->file.ops->goto_eof(db->file.handle)) == 0) {
        /* This is an empty file. Create a new fileheader unless the
         * user wanted a read-only version of the file
         */
        if (flags & COUCHSTORE_OPEN_FLAG_RDONLY) {
            error_pass(COUCHSTORE_ERROR_NO_HEADER);
        } else {
            error_pass(create_header(db));
        }
    } else {
        error_pass(find_header(db));
    }

    *pDb = db;
    return COUCHSTORE_SUCCESS;

cleanup:
    couchstore_close_db(db);
    return errcode;
}
Exemple #3
0
static int process_file(const char *file, int *total)
{
    Db *db;
    couchstore_error_t errcode;
    errcode = couchstore_open_db(file, COUCHSTORE_OPEN_FLAG_RDONLY, &db);
    if (errcode != COUCHSTORE_SUCCESS) {
        fprintf(stderr, "Failed to open \"%s\": %s\n",
                file, couchstore_strerror(errcode));
        return -1;
    }

    int count = 0;

    switch (mode) {
        case DumpBySequence:
            if (dumpTree) {
                errcode = couchstore_walk_seq_tree(db, 0, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                                   visit_node, &count);
            } else {
                errcode = couchstore_changes_since(db, 0, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                                   foldprint, &count);
            }
            break;
        case DumpByID:
            if (dumpTree) {
                errcode = couchstore_walk_id_tree(db, NULL, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                                  visit_node, &count);
            } else {
                errcode = couchstore_all_docs(db, NULL, COUCHSTORE_INCLUDE_CORRUPT_DOCS,
                                              foldprint, &count);
            }
            break;
    }
    (void)couchstore_close_db(db);

    if (errcode < 0) {
        fprintf(stderr, "Failed to dump database \"%s\": %s\n",
                file, couchstore_strerror(errcode));
        return -1;
    }

    *total += count;
    return 0;
}
Exemple #4
0
static int process_file(const char *file, int iterate_headers)
{
    Db *db;
    couchstore_error_t errcode;
    uint64_t btreesize = 0;

    errcode = couchstore_open_db(file, COUCHSTORE_OPEN_FLAG_RDONLY, &db);
    if (errcode != COUCHSTORE_SUCCESS) {
        fprintf(stderr, "Failed to open \"%s\": %s\n",
                file, couchstore_strerror(errcode));
        return -1;
    }

next_header:
    printf("DB Info (%s) - header at %"PRIu64"\n", file, db->header.position);
    printf("   file format version: %"PRIu64"\n", db->header.disk_version);
    printf("   update_seq: %"PRIu64"\n", db->header.update_seq);
    printf("   purge_seq: %"PRIu64"\n", db->header.purge_seq);
    print_db_info(db);
    if (db->header.by_id_root) {
        btreesize += db->header.by_id_root->subtreesize;
    }
    if (db->header.by_seq_root) {
        btreesize += db->header.by_seq_root->subtreesize;
    }
    printf("   B-tree size: %s\n", size_str(btreesize));
    printf("   total disk size: %s\n", size_str(db->file.pos));
    if (iterate_headers) {
        if (couchstore_rewind_db_header(db) == COUCHSTORE_SUCCESS) {
            printf("\n");
            goto next_header;
        }
    } else {
        couchstore_close_db(db);
    }

    return 0;
}
Exemple #5
0
void test_no_purge_items()
{
    int errcode, N;
    int redval;
    int ctx[2];
    int purge_sum[2] = {0, 0};
    Db *db = NULL;
    node_pointer *root = NULL, *newroot = NULL;
    couchfile_modify_request purge_rq;
    fprintf(stderr, "\nExecuting test_no_purge_items...\n");

    N = 211341;
    ctx[0] = N;
    ctx[1] = 1;
    remove(testpurgefile);
    try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    root = insert_items(&db->file, NULL, count_reduce, count_rereduce, N);
    assert(root != NULL);

    redval = red_intval(root, 0);
    assert(redval == N);
    fprintf(stderr, "Initial reduce value equals N\n");

    purge_rq = purge_request(&db->file, count_reduce, count_rereduce,
                    keepall_purge_kp, keepall_purge_kv, (void *) purge_sum);
    newroot = guided_purge_btree(&purge_rq, root, &errcode);

    assert(purge_sum[0] == 0 && purge_sum[1] == 0);
    fprintf(stderr, "guided_purge returned correct accumulator {0,0}\n");

    redval = red_intval(newroot, 0);
    assert(redval == N);
    fprintf(stderr, "Reduce value after guided purge equals N\n");

    try(iter_btree(&db->file, newroot, &ctx, check_vals_callback));
    fprintf(stderr, "Btree has same values after guided purge\n");

cleanup:
    free(root);
    if (root != newroot) {
        free(newroot);
    }
    couchstore_close_db(db);
    assert(errcode == 0);
}

void test_all_purge_items()
{
    int errcode, N;
    int redval;
    int purge_sum[2] = {0, 0};
    Db *db = NULL;
    node_pointer *root = NULL, *newroot = NULL;
    couchfile_modify_request purge_rq;
    fprintf(stderr, "\nExecuting test_all_purge_items...\n");

    N = 211341;
    remove(testpurgefile);
    try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    root = insert_items(&db->file, NULL, count_reduce, count_rereduce, N);
    assert(root != NULL);

    redval = red_intval(root, 0);
    assert(redval == N);
    fprintf(stderr, "Initial reduce value equals N\n");

    purge_rq = purge_request(&db->file, count_reduce, count_rereduce,
                            all_purge_kp, all_purge_kv, (void *) &purge_sum);
    newroot = guided_purge_btree(&purge_rq, root, &errcode);

    assert(purge_sum[0] == 0 && purge_sum[1] == N);
    fprintf(stderr, "guided_purge returned correct accumulator {0,N}\n");

    redval = red_intval(newroot, 0);
    assert(redval == 0);
    fprintf(stderr, "Reduce value after guided purge equals 0\n");

    assert(newroot == NULL);
    fprintf(stderr, "Btree is empty after guided purge\n");

cleanup:
    free(root);
    free(newroot);
    couchstore_close_db(db);
    assert(errcode == 0);
}


void test_partial_purge_items()
{
    int errcode, N;
    int exp_evenodd[2];
    int purge_count = 0;
    Db *db = NULL;
    node_pointer *root = NULL, *newroot = NULL;
    couchfile_modify_request purge_rq;
    fprintf(stderr, "\nExecuting test_partial_purge_items...\n");

    N = 211341;
    exp_evenodd[0] = N / 2;
    exp_evenodd[1] = N / 2 + N % 2;

    remove(testpurgefile);
    try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    root = insert_items(&db->file, NULL, evenodd_reduce, evenodd_rereduce, N);
    assert(root != NULL);

    assert(exp_evenodd[0] == red_intval(root, 0) && exp_evenodd[1] == red_intval(root, 1));
    fprintf(stderr, "Initial reduce value equals {NumEven, NumOdd}\n");

    purge_rq = purge_request(&db->file, evenodd_reduce, evenodd_rereduce,
                    evenodd_purge_kp, evenodd_purge_kv, (void *) &purge_count);
    newroot = guided_purge_btree(&purge_rq, root, &errcode);

    assert(purge_count == exp_evenodd[1]);
    fprintf(stderr, "guided_purge returned correct accumulator {0,NumOdd}\n");

    assert(red_intval(newroot, 0) == exp_evenodd[0] && red_intval(newroot, 1) == 0);
    fprintf(stderr, "Reduce value after guided purge equals {NumEven, 0}\n");

    try(iter_btree(&db->file, newroot, NULL, check_odd_callback));
    fprintf(stderr, "Btree has no odd values after guided purge\n");

cleanup:
    free(root);
    free(newroot);
    couchstore_close_db(db);
    assert(errcode == 0);
}

void test_partial_purge_items2()
{
    int errcode, N;
    Db *db = NULL;
    node_pointer *root = NULL, *newroot = NULL;
    int range_start, range_end;
    int count, purge_count = 0, iter_context = -1;
    couchfile_modify_request purge_rq;
    fprintf(stderr, "\nExecuting test_partial_purge_items2...\n");

    N = 320000;
    remove(testpurgefile);
    try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    root = insert_items(&db->file, NULL, uniq_reduce, uniq_rereduce, N);
    assert(root != NULL);

    count = red_intval(root, 1);
    range_start = red_intval(root, 2);
    range_end = red_intval(root, count + 1);
    assert(range_start == 0 && range_end == 63);

    fprintf(stderr, "Initial reduce value equals seq{0, 63}\n");

    purge_rq = purge_request(&db->file, uniq_reduce, uniq_rereduce,
                        skip_purge_kp, skip_purge_kv, (void *) &purge_count);
    newroot = guided_purge_btree(&purge_rq, root, &errcode);

    assert(purge_count == N / 2);
    fprintf(stderr, "guided_purge returned correct accumulator N/2\n");

    count = red_intval(newroot, 1);
    range_start = red_intval(newroot, 2);
    range_end = red_intval(newroot, count + 1);
    assert(red_intval(newroot, 0) == N / 2 && range_start == 0 && range_end == 31);
    fprintf(stderr, "Reduce value after guided purge equals {0, 31}\n");

    try(iter_btree(&db->file, newroot, &iter_context, check_skiprange_callback));
    fprintf(stderr, "Btree has only values within the range {0, 31} and keys are sorted\n");

cleanup:
    free(root);
    free(newroot);
    couchstore_close_db(db);
    assert(errcode == 0);
}

void test_partial_purge_with_stop()
{
    int errcode, N;
    int exp_evenodd[2];
    int purge_count = 0;
    Db *db = NULL;
    node_pointer *root = NULL, *newroot = NULL;
    couchfile_modify_request purge_rq;
    fprintf(stderr, "\nExecuting test_partial_purge_items...\n");

    N = 211341;
    exp_evenodd[0] = N / 2;
    exp_evenodd[1] = N / 2 + N % 2;

    remove(testpurgefile);
    try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    root = insert_items(&db->file, NULL, evenodd_reduce, evenodd_rereduce, N);
    assert(root != NULL);

    assert(exp_evenodd[0] == red_intval(root, 0) && exp_evenodd[1] == red_intval(root, 1));
    fprintf(stderr, "Initial reduce value equals {NumEven, NumOdd}\n");

    purge_rq = purge_request(&db->file, evenodd_reduce, evenodd_rereduce,
            evenodd_purge_kp, evenodd_stop_purge_kv, (void *) &purge_count);
    newroot = guided_purge_btree(&purge_rq, root, &errcode);

    assert(purge_count == 4);
    fprintf(stderr, "guided_purge returned correct accumulator - 4\n");

    assert(red_intval(newroot, 0) == exp_evenodd[0]);
    assert(red_intval(newroot, 1) == (exp_evenodd[1] - 4));
    fprintf(stderr, "Reduce value after guided purge equals {NumEven, NumOdd-4}\n");

    try(iter_btree(&db->file, newroot, NULL, check_odd_stop_callback));
    fprintf(stderr, "Btree does not contain first 4 odd values after guided purge\n");

cleanup:
    free(root);
    free(newroot);
    couchstore_close_db(db);
    assert(errcode == 0);
}

void test_add_remove_purge()
{
    int errcode, N, i;
    int exp_evenodd[2];
    int purge_count = 0;
    Db *db = NULL;
    node_pointer *root = NULL, *newroot = NULL;
    couchfile_modify_request purge_rq;
    int *arr = NULL;
    couchfile_modify_action *acts = NULL;
    sized_buf *keys = NULL;
    fprintf(stderr, "\nExecuting test_add_remove_purge...\n");

    N = 211341;
    exp_evenodd[0] = N / 2;
    exp_evenodd[1] = N / 2 + N % 2;

    remove(testpurgefile);
    try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    root = insert_items(&db->file, NULL, evenodd_reduce, evenodd_rereduce, N);
    assert(root != NULL);

    assert(exp_evenodd[0] == red_intval(root, 0) && exp_evenodd[1] == red_intval(root, 1));
    fprintf(stderr, "Initial reduce value equals {NumEven, NumOdd}\n");

    purge_rq = purge_request(&db->file, evenodd_reduce, evenodd_rereduce,
                    evenodd_purge_kp, evenodd_purge_kv, (void *) &purge_count);

    /* Add few add and remove actions in the modify request */
    arr = (int *) calloc(6, sizeof(int));
    keys = (sized_buf *) calloc(6, sizeof(sized_buf));
    acts = (couchfile_modify_action *) calloc(6, sizeof(couchfile_modify_action));

    arr[0] = 2;
    arr[1] = 4;
    arr[2] = 10;
    arr[3] = 14006;
    arr[4] = 200000;
    arr[5] = 500000;

    acts[0].type = ACTION_INSERT;
    acts[1].type = ACTION_REMOVE;
    acts[2].type = ACTION_REMOVE;
    acts[3].type = ACTION_INSERT;
    acts[4].type = ACTION_REMOVE;
    acts[5].type = ACTION_INSERT;


    for (i = 0; i < 6; i++) {
        keys[i].size  = sizeof(int);
        keys[i].buf = (void *) &arr[i];
        acts[i].key = &keys[i];
        acts[i].value.data = &keys[i];
    }

    purge_rq.actions = acts;
    purge_rq.num_actions = 6;
    purge_rq.enable_purging = 1;
    newroot = modify_btree(&purge_rq, root, &errcode);


    assert(purge_count == exp_evenodd[1]);
    fprintf(stderr, "Btree add_remove with purge returned correct purge_count - Numodds\n");

    assert(red_intval(newroot, 0) == (exp_evenodd[0] - 2) && red_intval(newroot, 1) == 0);
    fprintf(stderr, "Btree reduce value equals - {NumEven-2, 0}\n");

    try(iter_btree(&db->file, newroot, NULL, check_odd2_callback));
    fprintf(stderr, "Btree has no odd values after guided purge\n");
    fprintf(stderr, "Keys 4,10,200000 are not in tree after guided purge\n");

cleanup:
    free(root);
    free(newroot);
    free(keys);
    free(acts);
    free(arr);
    couchstore_close_db(db);
    assert(errcode == 0);
}

void test_only_single_leafnode()
{
    int errcode, N;
    int redval;
    int purge_sum[2] = {0,0};
    Db *db = NULL;
    node_pointer *root = NULL, *newroot = NULL;
    couchfile_modify_request purge_rq;

    fprintf(stderr, "\nExecuting test_only_single_leafnode...\n");
    N = 2;
    remove(testpurgefile);
    try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    root = insert_items(&db->file, NULL, count_reduce, count_rereduce, N);
    assert(root != NULL);

    redval = red_intval(root, 0);
    assert(redval == N);
    fprintf(stderr, "Initial reduce value equals N\n");

    purge_rq = purge_request(&db->file, count_reduce, count_rereduce, keepall_purge_kp, all_purge_kv, (void *) &purge_sum);
    newroot = guided_purge_btree(&purge_rq, root, &errcode);

    assert(purge_sum[0] == N && purge_sum[1] == 0);
    fprintf(stderr, "guided_purge returned correct accumulator {N,0}\n");

    redval = red_intval(newroot, 0);
    assert(redval == 0);
    fprintf(stderr, "Reduce value after guided purge equals 0\n");

    assert(newroot == NULL);
    fprintf(stderr, "Btree is empty after guided purge\n");

cleanup:
    free(root);
    free(newroot);
    couchstore_close_db(db);
    assert(errcode == 0);
}
Exemple #6
0
static int docset_check(Db *db, DocInfo *info, void *ctx)
{
    int errcode = 0;
    docset *ds = ctx;
    counterset *ctr = &ds->counters;
    ctr->totaldocs++;
    if (info->deleted) {
        ctr->deleted++;
    }
    EQUAL_INFO_BUF(id);
    EQUAL_INFO_BUF(rev_meta);
    Doc *doc;
    try(couchstore_open_doc_with_docinfo(db, info, &doc, DECOMPRESS_DOC_BODIES));
    if (testdocset.docs[testdocset.pos].data.size > 0) {
        assert(doc);
        EQUAL_DOC_BUF(data);
        EQUAL_DOC_BUF(id);
    }
    testdocset.pos++;
    couchstore_free_document(doc);
cleanup:
    assert(errcode == 0);
    return 0;
}

static int dociter_check(Db *db, DocInfo *info, void *ctx)
{
    int errcode = 0;
    docset *ds = ctx;
    counterset *ctr = &ds->counters;
    ctr->totaldocs++;
    if (info->deleted) {
        ctr->deleted++;
    }
    Doc *doc;
    try(couchstore_open_doc_with_docinfo(db, info, &doc, DECOMPRESS_DOC_BODIES));
    assert(doc);
    couchstore_free_document(doc);
cleanup:
    assert(errcode == 0);
    return 0;
}

static int dump_count(Db *db)
{
    int errcode = 0;
    ZERO(counters);
    try(couchstore_changes_since(db, 0, 0, counter_inc, &counters));
cleanup:
    assert(errcode == 0);
    return errcode;
}

static char zerometa[] = {0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3};

static void test_save_docs(int count, const char *doc_tpl)
{
    int errcode = 0;
    int i;
    char *idBuf, *valueBuf;
    Doc **docptrs;
    DocInfo **nfoptrs;
    sized_buf *ids = NULL;
    uint64_t idtreesize = 0;
    uint64_t seqtreesize = 0;
    uint64_t docssize = 0;
    uint64_t dbfilesize = 0;
    uint64_t *sequences = NULL;

    fprintf(stderr, "save_docs (doc count %d)... ", count);
    fflush(stderr);

    docset_init(count);
    srandom(0xdeadbeef);  // doc IDs should be consistent across runs
    for (i = 0; i < count; ++i) {
        idBuf = (char *) malloc(sizeof(char) * 32);
        assert(idBuf != NULL);
        int idsize = sprintf(idBuf, "doc%d-%lu", i, (unsigned long)random());
        valueBuf = (char *) malloc(sizeof(char) * (strlen(doc_tpl) + 20));
        assert(valueBuf != NULL);
        int valsize = sprintf(valueBuf, doc_tpl, i + 1);
        setdoc(&testdocset.docs[i], &testdocset.infos[i],
                idBuf, idsize, valueBuf, valsize, zerometa, sizeof(zerometa));
        testdocset.datasize += valsize;
    }

    docptrs = (Doc **) malloc(sizeof(Doc*) * count);
    assert(docptrs != NULL);
    for (i = 0; i < count; ++i) {
        docptrs[i] = &testdocset.docs[i];
    }

    nfoptrs = (DocInfo **) malloc(sizeof(DocInfo*) * count);
    assert(nfoptrs != NULL);
    for (i = 0; i < count; ++i) {
        nfoptrs[i] = &testdocset.infos[i];
    }

    unlink(testfilepath);
    Db *db;
    try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    assert(strcmp(couchstore_get_db_filename(db), testfilepath) == 0);
    try(couchstore_save_documents(db, docptrs, nfoptrs, count, 0));
    try(couchstore_commit(db));
    couchstore_close_db(db);

    try(couchstore_open_db(testfilepath, 0, &db));

    // Read back by doc ID:
    fprintf(stderr, "get by ID... ");
    testdocset.pos = 0;
    for (i = 0; i < count; ++i) {
        DocInfo* out_info;
        try(couchstore_docinfo_by_id(db, testdocset.docs[i].id.buf, testdocset.docs[i].id.size,
                                     &out_info));
        docset_check(db, out_info, &testdocset);
        couchstore_free_docinfo(out_info);
    }

    // Read back in bulk by doc ID:
    fprintf(stderr, "bulk IDs... ");
    ids = malloc(count * sizeof(sized_buf));
    for (i = 0; i < count; ++i) {
        ids[i] = docptrs[i]->id;
    }
    ZERO(testdocset.counters);
    try(couchstore_docinfos_by_id(db, ids, count, 0, dociter_check, &testdocset));
    assert(testdocset.counters.totaldocs == count);
    assert(testdocset.counters.deleted == 0);

    // Read back by sequence:
    fprintf(stderr, "get by sequence... ");
    sequences = malloc(count * sizeof(*sequences));
    testdocset.pos = 0;
    for (i = 0; i < count; ++i) {
        DocInfo* out_info;
        sequences[i] = testdocset.infos[i].db_seq;
        assert(sequences[i] == (uint64_t)i + 1);
        try(couchstore_docinfo_by_sequence(db, testdocset.infos[i].db_seq, &out_info));
        docset_check(db, out_info, &testdocset);
        couchstore_free_docinfo(out_info);
    }

    // Read back in bulk by sequence:
    fprintf(stderr, "bulk sequences... ");
    testdocset.pos = 0;
    ZERO(testdocset.counters);
    try(couchstore_docinfos_by_sequence(db, sequences, count, 0, docset_check, &testdocset));
    assert(testdocset.counters.totaldocs == count);
    assert(testdocset.counters.deleted == 0);

    // Read back using changes_since:
    fprintf(stderr, "changes_since... ");
    testdocset.pos = 0;
    ZERO(testdocset.counters);
    try(couchstore_changes_since(db, 0, 0, docset_check, &testdocset));
    assert(testdocset.counters.totaldocs == count);
    assert(testdocset.counters.deleted == 0);

    idtreesize = db->header.by_id_root->subtreesize;
    seqtreesize = db->header.by_seq_root->subtreesize;
    const raw_by_id_reduce *reduce = (const raw_by_id_reduce*)db->header.by_id_root->reduce_value.buf;
    docssize = decode_raw48(reduce->size);
    dbfilesize = db->file.pos;

    assert(dbfilesize > 0);
    assert(idtreesize > 0);
    assert(seqtreesize > 0);
    assert(docssize > 0);
    assert(idtreesize < dbfilesize);
    assert(seqtreesize < dbfilesize);
    assert(docssize < dbfilesize);
    assert(db->header.local_docs_root == NULL);
    assert((idtreesize + seqtreesize + docssize) < dbfilesize);

    couchstore_close_db(db);
cleanup:
    free(ids);
    free(sequences);
    for (i = 0; i < count; ++i) {
        free(docptrs[i]->id.buf);
        free(docptrs[i]->data.buf);
    }
    free(docptrs);
    free(nfoptrs);
    assert(errcode == 0);
}

static void test_save_doc(void)
{
    fprintf(stderr, "save_doc... ");
    fflush(stderr);
    int errcode = 0;
    docset_init(4);
    SETDOC(0, "doc1", "{\"test_doc_index\":1}", zerometa);
    SETDOC(1, "doc2", "{\"test_doc_index\":2}", zerometa);
    SETDOC(2, "doc3", "{\"test_doc_index\":3}", zerometa);
    SETDOC(3, "doc4", "{\"test_doc_index\":4}", zerometa);
    unlink(testfilepath);
    Db *db;
    try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    try(couchstore_save_document(db, &testdocset.docs[0],
                                     &testdocset.infos[0], 0));
    try(couchstore_save_document(db, &testdocset.docs[1],
                                     &testdocset.infos[1], 0));
    try(couchstore_save_document(db, &testdocset.docs[2],
                                     &testdocset.infos[2], 0));
    try(couchstore_save_document(db, &testdocset.docs[3],
                                     &testdocset.infos[3], 0));
    try(couchstore_commit(db));
    couchstore_close_db(db);

    // Check that sequence numbers got filled in
    unsigned i;
    for (i = 0; i < 4; ++i) {
        assert(testdocset.infos[i].db_seq == i + 1);
    }

    //Read back
    try(couchstore_open_db(testfilepath, 0, &db));
    try(couchstore_changes_since(db, 0, 0, docset_check, &testdocset));
    assert(testdocset.counters.totaldocs == 4);
    assert(testdocset.counters.deleted == 0);

    DbInfo info;
    assert(couchstore_db_info(db, &info) == COUCHSTORE_SUCCESS);
    assert(info.last_sequence == 4);
    assert(info.doc_count == 4);
    assert(info.deleted_count == 0);
    assert(info.header_position == 4096);

    couchstore_close_db(db);
cleanup:
    assert(errcode == 0);
}

static void test_compressed_doc_body(void)
{
    fprintf(stderr, "compressed bodies... ");
    fflush(stderr);
    int errcode = 0;
    docset_init(2);
    SETDOC(0, "doc1", "{\"test_doc_index\":1, \"val\":\"blah blah blah blah blah blah\"}", zerometa);
    SETDOC(1, "doc2", "{\"test_doc_index\":2, \"val\":\"blah blah blah blah blah blah\"}", zerometa);
    Doc *docptrs [2] =  { &testdocset.docs[0],
                          &testdocset.docs[1]
                        };
    DocInfo *nfoptrs [2] =  { &testdocset.infos[0],
                              &testdocset.infos[1]
                            };
    testdocset.infos[1].content_meta = COUCH_DOC_IS_COMPRESSED; //Mark doc2 as to be snappied.
    unlink(testfilepath);
    Db *db;
    try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    try(couchstore_save_documents(db, docptrs, nfoptrs, 2,
                                      COMPRESS_DOC_BODIES));
    try(couchstore_commit(db));
    couchstore_close_db(db);
    //Read back
    try(couchstore_open_db(testfilepath, 0, &db));
    try(couchstore_changes_since(db, 0, 0, docset_check, &testdocset));
    assert(testdocset.counters.totaldocs == 2);
    assert(testdocset.counters.deleted == 0);
    couchstore_close_db(db);
cleanup:
    assert(errcode == 0);
}

static void test_dump_empty_db(void)
{
    fprintf(stderr, "dump empty db... ");
    fflush(stderr);
    unlink(testfilepath);
    Db *db;
    couchstore_error_t errcode;
    
    try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    try(couchstore_close_db(db));
    try(couchstore_open_db(testfilepath, 0, &db));
    dump_count(db);
    assert(counters.totaldocs == 0);
    assert(counters.deleted == 0);

    DbInfo info;
    assert(couchstore_db_info(db, &info) == COUCHSTORE_SUCCESS);
    assert(strcmp(info.filename, testfilepath) == 0);
    assert(info.last_sequence == 0);
    assert(info.doc_count == 0);
    assert(info.deleted_count == 0);
    assert(info.space_used == 0);
    assert(info.header_position == 0);

    couchstore_close_db(db);
cleanup:
    assert(errcode == 0);
}

static void test_local_docs(void)
{
    fprintf(stderr, "local docs... ");
    fflush(stderr);
    int errcode = 0;
    Db *db;
    LocalDoc lDocWrite;
    LocalDoc *lDocRead = NULL;
    unlink(testfilepath);
    try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    lDocWrite.id.buf = "_local/testlocal";
    lDocWrite.id.size = 16;
    lDocWrite.json.buf = "{\"test\":true}";
    lDocWrite.json.size = 13;
    lDocWrite.deleted = 0;
    couchstore_save_local_document(db, &lDocWrite);
    couchstore_commit(db);
    couchstore_close_db(db);
    couchstore_open_db(testfilepath, 0, &db);
    couchstore_open_local_document(db, "_local/testlocal", 16, &lDocRead);
    assert(lDocRead);
    assert(lDocRead->json.size == 13);
    assert(memcmp(lDocRead->json.buf, "{\"test\":true}", 13) == 0);
    couchstore_free_local_document(lDocRead);
    couchstore_close_db(db);
cleanup:
    assert(errcode == 0);
}

static void test_open_file_error(void)
{
    fprintf(stderr, "opening nonexistent file errors... ");
    fflush(stderr);
    unlink(testfilepath);
    Db *db;
    int errcode = couchstore_open_db(testfilepath, 0, &db);

    if(errcode != 0) {
        print_os_err();
    }

    assert(errcode == COUCHSTORE_ERROR_NO_SUCH_FILE);

    // make sure os.c didn't accidentally call close(0):
    assert(lseek(0, 0, SEEK_CUR) >= 0 || errno != EBADF);
}

static void shuffle(Doc **docs, DocInfo **docinfos, size_t n)
{
    if (n > 1) {
        size_t i;
        for (i = 0; i < n - 1; i++) {
          size_t j = i + rand() / (RAND_MAX / (n - i) + 1);
          DocInfo *docinfo;
          Doc *doc = docs[j];
          docs[j] = docs[i];
          docs[i] = doc;

          docinfo = docinfos[j];
          docinfos[j] = docinfos[i];
          docinfos[i] = docinfo;
        }
    }
}

static int docmap_check(Db *db, DocInfo *info, void *ctx)
{
    (void)db;
    char* docmap = (char*)ctx;
    int i;
    char buffer[100];
    memcpy(buffer, info->id.buf, info->id.size);
    buffer[info->id.size] = 0; // null terminate
    sscanf(buffer, "doc%d", &i);
    assert(docmap[i] == 0);
    docmap[i] = 1;
    return 0;
}

static void test_changes_no_dups(void)
{
    fprintf(stderr, "changes no dupes... ");
    fflush(stderr);
    int errcode = 0;
    int i;
    const int numdocs = 10000;
    int updatebatch = 1000;
    Doc **docptrs;
    DocInfo **nfoptrs;
    char *docmap;
    Db *db;
    docset_init(numdocs);
    for (i=0; i < numdocs; i++) {
        char* id = malloc(100);
        char* body = malloc(100);
        sprintf(id, "doc%d", i);
        sprintf(body, "{\"test_doc_index\":%d}", i);
        setdoc(&testdocset.docs[i], &testdocset.infos[i],
                id, strlen(id),
                body, strlen(body),
                zerometa, sizeof(zerometa));
    }
    docptrs = malloc(numdocs * sizeof(Doc*));
    nfoptrs = malloc(numdocs * sizeof(DocInfo*));
    docmap = malloc(numdocs);
    for (i=0; i < numdocs; i++) {
        docptrs[i] = &testdocset.docs[i];
        nfoptrs[i] = &testdocset.infos[i];
    }
    unlink(testfilepath);
    try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
    // only save half the docs at first.
    try(couchstore_save_documents(db, docptrs, nfoptrs, numdocs/2, 0));
    try(couchstore_commit(db));
    couchstore_close_db(db);

    for (i=0; i < numdocs/2; i++) {
        // increment the rev for already added docs
        nfoptrs[i]->rev_seq++;
    }
    srand(10); // make deterministic
    // now shuffle so some bulk updates contain previous docs and new docs
    shuffle(docptrs, nfoptrs, numdocs);
    try(couchstore_open_db(testfilepath, 0, &db));
    for (i=0; i < numdocs; i += updatebatch) {
        // now do bulk updates and check the changes for dups
        try(couchstore_save_documents(db, docptrs + i, nfoptrs + i, updatebatch, 0));
        try(couchstore_commit(db));
        memset(docmap, 0, numdocs);
        try(couchstore_changes_since(db, 0, 0, docmap_check, docmap));
    }

    DbInfo info;
    assert(couchstore_db_info(db, &info) == COUCHSTORE_SUCCESS);
    assert(info.last_sequence == (uint64_t)(numdocs + numdocs/2));
    assert(info.doc_count == (uint64_t)numdocs);
    assert(info.deleted_count == 0);

    couchstore_close_db(db);
cleanup:
    for (i=0; i < numdocs; i++) {
        free(docptrs[i]->id.buf);
        free(docptrs[i]->data.buf);
    }
    free(docptrs);
    free(nfoptrs);
    free(docmap);
    assert(errcode == 0);
}


static void mb5086(void)
{
    Db *db;
    Doc d;
    DocInfo i;
    couchstore_error_t err;

    fprintf(stderr, "regression mb-5086.... ");
    fflush(stderr);

    setdoc(&d, &i, "hi", 2, "foo", 3, NULL, 0);
    err = couchstore_open_db("mb5085.couch", COUCHSTORE_OPEN_FLAG_CREATE, &db);
    assert(err == COUCHSTORE_SUCCESS);
    assert(couchstore_save_document(db, &d, &i, 0) == COUCHSTORE_SUCCESS);
    assert(couchstore_commit(db) == COUCHSTORE_SUCCESS);
    assert(couchstore_close_db(db) == COUCHSTORE_SUCCESS);
    assert(remove("mb5085.couch") == 0);
}

static void test_asis_seqs(void)
{
   Db *db = NULL;
   Doc d;
   DocInfo i;
   DocInfo *ir;
   couchstore_error_t errcode;

   fprintf(stderr, "as-is seqs.... ");
   fflush(stderr);

   try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
   setdoc(&d, &i, "test", 4, "foo", 3, NULL, 0);
   i.db_seq = 1;
   try(couchstore_save_document(db, &d, &i, COUCHSTORE_SEQUENCE_AS_IS));
   assert(db->header.update_seq == 1);

   setdoc(&d, &i, "test_two", 8, "foo", 3, NULL, 0);
   i.db_seq = 12;
   try(couchstore_save_document(db, &d, &i, COUCHSTORE_SEQUENCE_AS_IS));
   assert(db->header.update_seq == 12);

   setdoc(&d, &i, "test_foo", 8, "foo", 3, NULL, 0);
   i.db_seq = 6;
   try(couchstore_save_document(db, &d, &i, COUCHSTORE_SEQUENCE_AS_IS));
   assert(db->header.update_seq == 12);

   try(couchstore_docinfo_by_id(db, "test", 4, &ir));
   assert(ir->db_seq == 1);
   couchstore_free_docinfo(ir);
   
   try(couchstore_docinfo_by_id(db, "test_two", 8, &ir));
   assert(ir->db_seq == 12);
   couchstore_free_docinfo(ir);

   try(couchstore_docinfo_by_id(db, "test_foo", 8, &ir));
   assert(ir->db_seq == 6);
   couchstore_free_docinfo(ir);

cleanup:
   if (db != NULL) {
       couchstore_close_db(db);
   }
   assert(errcode == COUCHSTORE_SUCCESS);
}

static void test_huge_revseq(void)
{
    Db *db;
    Doc d;
    DocInfo i;
    DocInfo *i2;
    couchstore_error_t err;

    fprintf(stderr, "huge rev_seq.... ");
    fflush(stderr);

    setdoc(&d, &i, "hi", 2, "foo", 3, NULL, 0);
    i.rev_seq = 5294967296;

    err = couchstore_open_db("bigrevseq.couch", COUCHSTORE_OPEN_FLAG_CREATE, &db);
    assert(err == COUCHSTORE_SUCCESS);
    assert(couchstore_save_document(db, &d, &i, 0) == COUCHSTORE_SUCCESS);
    assert(couchstore_commit(db) == COUCHSTORE_SUCCESS);
    assert(couchstore_docinfo_by_id(db, "hi", 2, &i2) == COUCHSTORE_SUCCESS);
    assert(i2->rev_seq == 5294967296);
    couchstore_free_docinfo(i2);
    assert(couchstore_close_db(db) == COUCHSTORE_SUCCESS);
    assert(remove("bigrevseq.couch") == 0);
}

static void test_dropped_handle(void)
{
   couchstore_error_t errcode;
   Db* db = NULL;
   Doc d;
   DocInfo i;

   fprintf(stderr, "drop file handle.... ");
   fflush(stderr);

   try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
   setdoc(&d, &i, "test", 4, "foo", 3, NULL, 0);
   try(couchstore_save_document(db, &d, &i, 0));
   try(couchstore_commit(db));

   try(couchstore_drop_file(db));
   assert(couchstore_save_document(db, &d, &i, 0) == COUCHSTORE_ERROR_FILE_CLOSED);

   try(couchstore_reopen_file(db, testfilepath, 0));

   Doc* rd;
   try(couchstore_open_document(db, "test", 4, &rd, 0));
   couchstore_free_document(rd);
cleanup:
   if (db != NULL) {
       couchstore_close_db(db);
   }
   assert(errcode == COUCHSTORE_SUCCESS);
}

int main(int argc, const char *argv[])
{
    int doc_counts[] = { 4, 69, 666, 9090 };
    unsigned i;
    const char *small_doc_tpl = "{\"test_doc_index\":%d}";
    const char *large_doc_tpl =
        "{"
        "\"test_doc_index\":%d,"
        "\"field1\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\","
        "\"field2\": \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\","
        "\"field3\": \"cccccccccccccccccccccccccccccccccccccccccccccccccc"
        "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
        "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
        "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
        "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
        "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
        "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\""
        "}";

    file_merger_tests();
    file_sorter_tests();

    if (argc > 1)
        strcpy(testfilepath, argv[1]);
    printf("Using test database at %s\n", testfilepath);
    
    test_bitfield_fns();

    test_open_file_error();
    fprintf(stderr, "OK \n");
    test_dump_empty_db();
    fprintf(stderr, " OK\n");
    test_save_doc();
    fprintf(stderr, " OK\n");
    for (i = 0; i < (sizeof(doc_counts) / sizeof(int)); ++i) {
        test_save_docs(doc_counts[i], small_doc_tpl);
        fprintf(stderr, " OK\n");
        test_save_docs(doc_counts[i], large_doc_tpl);
        fprintf(stderr, " OK\n");
    }
    test_local_docs();
    fprintf(stderr, " OK\n");
    test_compressed_doc_body();
    fprintf(stderr, " OK\n");
    test_changes_no_dups();
    fprintf(stderr, " OK\n");
    mb5086();
    fprintf(stderr, " OK\n");
    unlink(testfilepath);
    test_huge_revseq();
    fprintf(stderr, " OK\n");
    unlink(testfilepath);
    test_asis_seqs();
    fprintf(stderr, " OK\n");
    unlink(testfilepath);
    test_dropped_handle();
    fprintf(stderr, " OK\n");
    unlink(testfilepath);

    // make sure os.c didn't accidentally call close(0):
    assert(lseek(0, 0, SEEK_CUR) >= 0 || errno != EBADF);

    mapreduce_tests();
    view_tests();

    return 0;
}