/* Helper function to populate a btree with {1..N} sequence */
static node_pointer *insert_items(tree_file *file, node_pointer *root,
reduce_fn reduce,
reduce_fn rereduce,
int n)
{
int errcode, i;
couchfile_modify_request rq;
couchfile_modify_action *acts;
node_pointer *nroot = NULL;
int *arr1, *arr2;
sized_buf *keys, *vals;
arr1 = (int *) calloc(n, sizeof(int));
arr2 = (int *) calloc(n, sizeof(int));
keys = (sized_buf *) calloc(n, sizeof(sized_buf));
vals = (sized_buf *) calloc(n, sizeof(sized_buf));
acts = (couchfile_modify_action *) calloc(n, sizeof(couchfile_modify_action));
for (i = 0; i < n; i++) {
arr1[i] = i + 1;
arr2[i] = (i + 1) % 64;
keys[i].size = sizeof(int);
keys[i].buf = (void *) &arr1[i];
vals[i].size = sizeof(int);
vals[i].buf = (void *) &arr2[i];
acts[i].type = ACTION_INSERT;
acts[i].value.data = &vals[i];
acts[i].key = &keys[i];
}
rq.cmp.compare = int_cmp;
rq.file = file;
rq.actions = acts;
rq.num_actions = n;
rq.reduce = reduce;
rq.rereduce = rereduce;
rq.compacting = 0;
rq.kv_chunk_threshold = 6 * 1024;
rq.kp_chunk_threshold = 6 * 1024;
nroot = modify_btree(&rq, root, &errcode);
free(arr1);
free(arr2);
free(keys);
free(vals);
free(acts);
assert(errcode == 0);
return nroot;
}
LIBCOUCHSTORE_API
couchstore_error_t couchstore_save_local_document(Db *db, LocalDoc *lDoc)
{
couchstore_error_t errcode;
couchfile_modify_action ldupdate;
sized_buf cmptmp;
node_pointer *nroot = NULL;
if (lDoc->deleted) {
ldupdate.type = ACTION_REMOVE;
} else {
ldupdate.type = ACTION_INSERT;
}
ldupdate.key = &lDoc->id;
ldupdate.value.data = &lDoc->json;
couchfile_modify_request rq;
rq.cmp.compare = ebin_cmp;
rq.cmp.arg = &cmptmp;
rq.num_actions = 1;
rq.actions = &ldupdate;
rq.fetch_callback = NULL;
rq.reduce = NULL;
rq.rereduce = NULL;
rq.file = &db->file;
rq.compacting = 0;
nroot = modify_btree(&rq, db->header.local_docs_root, &errcode);
if (errcode == COUCHSTORE_SUCCESS && nroot != db->header.local_docs_root) {
free(db->header.local_docs_root);
db->header.local_docs_root = nroot;
}
return errcode;
}
void test_no_purge_items()
{
int errcode, N;
int redval;
int ctx[2];
int purge_sum[2] = {0, 0};
Db *db = NULL;
node_pointer *root = NULL, *newroot = NULL;
couchfile_modify_request purge_rq;
fprintf(stderr, "\nExecuting test_no_purge_items...\n");
N = 211341;
ctx[0] = N;
ctx[1] = 1;
remove(testpurgefile);
try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
root = insert_items(&db->file, NULL, count_reduce, count_rereduce, N);
assert(root != NULL);
redval = red_intval(root, 0);
assert(redval == N);
fprintf(stderr, "Initial reduce value equals N\n");
purge_rq = purge_request(&db->file, count_reduce, count_rereduce,
keepall_purge_kp, keepall_purge_kv, (void *) purge_sum);
newroot = guided_purge_btree(&purge_rq, root, &errcode);
assert(purge_sum[0] == 0 && purge_sum[1] == 0);
fprintf(stderr, "guided_purge returned correct accumulator {0,0}\n");
redval = red_intval(newroot, 0);
assert(redval == N);
fprintf(stderr, "Reduce value after guided purge equals N\n");
try(iter_btree(&db->file, newroot, &ctx, check_vals_callback));
fprintf(stderr, "Btree has same values after guided purge\n");
cleanup:
free(root);
if (root != newroot) {
free(newroot);
}
couchstore_close_db(db);
assert(errcode == 0);
}
void test_all_purge_items()
{
int errcode, N;
int redval;
int purge_sum[2] = {0, 0};
Db *db = NULL;
node_pointer *root = NULL, *newroot = NULL;
couchfile_modify_request purge_rq;
fprintf(stderr, "\nExecuting test_all_purge_items...\n");
N = 211341;
remove(testpurgefile);
try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
root = insert_items(&db->file, NULL, count_reduce, count_rereduce, N);
assert(root != NULL);
redval = red_intval(root, 0);
assert(redval == N);
fprintf(stderr, "Initial reduce value equals N\n");
purge_rq = purge_request(&db->file, count_reduce, count_rereduce,
all_purge_kp, all_purge_kv, (void *) &purge_sum);
newroot = guided_purge_btree(&purge_rq, root, &errcode);
assert(purge_sum[0] == 0 && purge_sum[1] == N);
fprintf(stderr, "guided_purge returned correct accumulator {0,N}\n");
redval = red_intval(newroot, 0);
assert(redval == 0);
fprintf(stderr, "Reduce value after guided purge equals 0\n");
assert(newroot == NULL);
fprintf(stderr, "Btree is empty after guided purge\n");
cleanup:
free(root);
free(newroot);
couchstore_close_db(db);
assert(errcode == 0);
}
void test_partial_purge_items()
{
int errcode, N;
int exp_evenodd[2];
int purge_count = 0;
Db *db = NULL;
node_pointer *root = NULL, *newroot = NULL;
couchfile_modify_request purge_rq;
fprintf(stderr, "\nExecuting test_partial_purge_items...\n");
N = 211341;
exp_evenodd[0] = N / 2;
exp_evenodd[1] = N / 2 + N % 2;
remove(testpurgefile);
try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
root = insert_items(&db->file, NULL, evenodd_reduce, evenodd_rereduce, N);
assert(root != NULL);
assert(exp_evenodd[0] == red_intval(root, 0) && exp_evenodd[1] == red_intval(root, 1));
fprintf(stderr, "Initial reduce value equals {NumEven, NumOdd}\n");
purge_rq = purge_request(&db->file, evenodd_reduce, evenodd_rereduce,
evenodd_purge_kp, evenodd_purge_kv, (void *) &purge_count);
newroot = guided_purge_btree(&purge_rq, root, &errcode);
assert(purge_count == exp_evenodd[1]);
fprintf(stderr, "guided_purge returned correct accumulator {0,NumOdd}\n");
assert(red_intval(newroot, 0) == exp_evenodd[0] && red_intval(newroot, 1) == 0);
fprintf(stderr, "Reduce value after guided purge equals {NumEven, 0}\n");
try(iter_btree(&db->file, newroot, NULL, check_odd_callback));
fprintf(stderr, "Btree has no odd values after guided purge\n");
cleanup:
free(root);
free(newroot);
couchstore_close_db(db);
assert(errcode == 0);
}
void test_partial_purge_items2()
{
int errcode, N;
Db *db = NULL;
node_pointer *root = NULL, *newroot = NULL;
int range_start, range_end;
int count, purge_count = 0, iter_context = -1;
couchfile_modify_request purge_rq;
fprintf(stderr, "\nExecuting test_partial_purge_items2...\n");
N = 320000;
remove(testpurgefile);
try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
root = insert_items(&db->file, NULL, uniq_reduce, uniq_rereduce, N);
assert(root != NULL);
count = red_intval(root, 1);
range_start = red_intval(root, 2);
range_end = red_intval(root, count + 1);
assert(range_start == 0 && range_end == 63);
fprintf(stderr, "Initial reduce value equals seq{0, 63}\n");
purge_rq = purge_request(&db->file, uniq_reduce, uniq_rereduce,
skip_purge_kp, skip_purge_kv, (void *) &purge_count);
newroot = guided_purge_btree(&purge_rq, root, &errcode);
assert(purge_count == N / 2);
fprintf(stderr, "guided_purge returned correct accumulator N/2\n");
count = red_intval(newroot, 1);
range_start = red_intval(newroot, 2);
range_end = red_intval(newroot, count + 1);
assert(red_intval(newroot, 0) == N / 2 && range_start == 0 && range_end == 31);
fprintf(stderr, "Reduce value after guided purge equals {0, 31}\n");
try(iter_btree(&db->file, newroot, &iter_context, check_skiprange_callback));
fprintf(stderr, "Btree has only values within the range {0, 31} and keys are sorted\n");
cleanup:
free(root);
free(newroot);
couchstore_close_db(db);
assert(errcode == 0);
}
void test_partial_purge_with_stop()
{
int errcode, N;
int exp_evenodd[2];
int purge_count = 0;
Db *db = NULL;
node_pointer *root = NULL, *newroot = NULL;
couchfile_modify_request purge_rq;
fprintf(stderr, "\nExecuting test_partial_purge_items...\n");
N = 211341;
exp_evenodd[0] = N / 2;
exp_evenodd[1] = N / 2 + N % 2;
remove(testpurgefile);
try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
root = insert_items(&db->file, NULL, evenodd_reduce, evenodd_rereduce, N);
assert(root != NULL);
assert(exp_evenodd[0] == red_intval(root, 0) && exp_evenodd[1] == red_intval(root, 1));
fprintf(stderr, "Initial reduce value equals {NumEven, NumOdd}\n");
purge_rq = purge_request(&db->file, evenodd_reduce, evenodd_rereduce,
evenodd_purge_kp, evenodd_stop_purge_kv, (void *) &purge_count);
newroot = guided_purge_btree(&purge_rq, root, &errcode);
assert(purge_count == 4);
fprintf(stderr, "guided_purge returned correct accumulator - 4\n");
assert(red_intval(newroot, 0) == exp_evenodd[0]);
assert(red_intval(newroot, 1) == (exp_evenodd[1] - 4));
fprintf(stderr, "Reduce value after guided purge equals {NumEven, NumOdd-4}\n");
try(iter_btree(&db->file, newroot, NULL, check_odd_stop_callback));
fprintf(stderr, "Btree does not contain first 4 odd values after guided purge\n");
cleanup:
free(root);
free(newroot);
couchstore_close_db(db);
assert(errcode == 0);
}
void test_add_remove_purge()
{
int errcode, N, i;
int exp_evenodd[2];
int purge_count = 0;
Db *db = NULL;
node_pointer *root = NULL, *newroot = NULL;
couchfile_modify_request purge_rq;
int *arr = NULL;
couchfile_modify_action *acts = NULL;
sized_buf *keys = NULL;
fprintf(stderr, "\nExecuting test_add_remove_purge...\n");
N = 211341;
exp_evenodd[0] = N / 2;
exp_evenodd[1] = N / 2 + N % 2;
remove(testpurgefile);
try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
root = insert_items(&db->file, NULL, evenodd_reduce, evenodd_rereduce, N);
assert(root != NULL);
assert(exp_evenodd[0] == red_intval(root, 0) && exp_evenodd[1] == red_intval(root, 1));
fprintf(stderr, "Initial reduce value equals {NumEven, NumOdd}\n");
purge_rq = purge_request(&db->file, evenodd_reduce, evenodd_rereduce,
evenodd_purge_kp, evenodd_purge_kv, (void *) &purge_count);
/* Add few add and remove actions in the modify request */
arr = (int *) calloc(6, sizeof(int));
keys = (sized_buf *) calloc(6, sizeof(sized_buf));
acts = (couchfile_modify_action *) calloc(6, sizeof(couchfile_modify_action));
arr[0] = 2;
arr[1] = 4;
arr[2] = 10;
arr[3] = 14006;
arr[4] = 200000;
arr[5] = 500000;
acts[0].type = ACTION_INSERT;
acts[1].type = ACTION_REMOVE;
acts[2].type = ACTION_REMOVE;
acts[3].type = ACTION_INSERT;
acts[4].type = ACTION_REMOVE;
acts[5].type = ACTION_INSERT;
for (i = 0; i < 6; i++) {
keys[i].size = sizeof(int);
keys[i].buf = (void *) &arr[i];
acts[i].key = &keys[i];
acts[i].value.data = &keys[i];
}
purge_rq.actions = acts;
purge_rq.num_actions = 6;
purge_rq.enable_purging = 1;
newroot = modify_btree(&purge_rq, root, &errcode);
assert(purge_count == exp_evenodd[1]);
fprintf(stderr, "Btree add_remove with purge returned correct purge_count - Numodds\n");
assert(red_intval(newroot, 0) == (exp_evenodd[0] - 2) && red_intval(newroot, 1) == 0);
fprintf(stderr, "Btree reduce value equals - {NumEven-2, 0}\n");
try(iter_btree(&db->file, newroot, NULL, check_odd2_callback));
fprintf(stderr, "Btree has no odd values after guided purge\n");
fprintf(stderr, "Keys 4,10,200000 are not in tree after guided purge\n");
cleanup:
free(root);
free(newroot);
free(keys);
free(acts);
free(arr);
couchstore_close_db(db);
assert(errcode == 0);
}
void test_only_single_leafnode()
{
int errcode, N;
int redval;
int purge_sum[2] = {0,0};
Db *db = NULL;
node_pointer *root = NULL, *newroot = NULL;
couchfile_modify_request purge_rq;
fprintf(stderr, "\nExecuting test_only_single_leafnode...\n");
N = 2;
remove(testpurgefile);
try(couchstore_open_db(testpurgefile, COUCHSTORE_OPEN_FLAG_CREATE, &db));
root = insert_items(&db->file, NULL, count_reduce, count_rereduce, N);
assert(root != NULL);
redval = red_intval(root, 0);
assert(redval == N);
fprintf(stderr, "Initial reduce value equals N\n");
purge_rq = purge_request(&db->file, count_reduce, count_rereduce, keepall_purge_kp, all_purge_kv, (void *) &purge_sum);
newroot = guided_purge_btree(&purge_rq, root, &errcode);
assert(purge_sum[0] == N && purge_sum[1] == 0);
fprintf(stderr, "guided_purge returned correct accumulator {N,0}\n");
redval = red_intval(newroot, 0);
assert(redval == 0);
fprintf(stderr, "Reduce value after guided purge equals 0\n");
assert(newroot == NULL);
fprintf(stderr, "Btree is empty after guided purge\n");
cleanup:
free(root);
free(newroot);
couchstore_close_db(db);
assert(errcode == 0);
}
static couchstore_error_t update_indexes(Db *db,
sized_buf *seqs,
sized_buf *seqvals,
sized_buf *ids,
sized_buf *idvals,
int numdocs)
{
couchfile_modify_action *idacts;
couchfile_modify_action *seqacts;
const sized_buf **sorted_ids = NULL;
size_t size;
fatbuf *actbuf;
node_pointer *new_id_root;
node_pointer *new_seq_root;
couchstore_error_t errcode;
couchfile_modify_request seqrq, idrq;
sized_buf tmpsb;
int ii;
/*
** Two action list up to numdocs * 2 in size + Compare keys for ids,
** and compare keys for removed seqs found from id index +
** Max size of a int64 erlang term (for deleted seqs)
*/
size = 4 * sizeof(couchfile_modify_action) + 2 * sizeof(sized_buf) + 10;
actbuf = fatbuf_alloc(numdocs * size);
error_unless(actbuf, COUCHSTORE_ERROR_ALLOC_FAIL);
idacts = fatbuf_get(actbuf, numdocs * sizeof(couchfile_modify_action) * 2);
seqacts = fatbuf_get(actbuf, numdocs * sizeof(couchfile_modify_action) * 2);
error_unless(idacts && seqacts, COUCHSTORE_ERROR_ALLOC_FAIL);
index_update_ctx fetcharg = {
seqacts, 0, &seqs, &seqvals, 0, actbuf
};
// Sort the array indexes of ids[] by ascending id. Since we can't pass context info to qsort,
// actually sort an array of pointers to the elements of ids[], rather than the array indexes.
sorted_ids = malloc(numdocs * sizeof(sized_buf*));
error_unless(sorted_ids, COUCHSTORE_ERROR_ALLOC_FAIL);
for (ii = 0; ii < numdocs; ++ii) {
sorted_ids[ii] = &ids[ii];
}
qsort(sorted_ids, numdocs, sizeof(sorted_ids[0]), &ebin_ptr_compare);
// Assemble idacts[] array, in sorted order by id:
for (ii = 0; ii < numdocs; ii++) {
ptrdiff_t isorted = sorted_ids[ii] - ids; // recover index of ii'th id in sort order
idacts[ii * 2].type = ACTION_FETCH;
idacts[ii * 2].value.arg = &fetcharg;
idacts[ii * 2 + 1].type = ACTION_INSERT;
idacts[ii * 2 + 1].value.data = &idvals[isorted];
idacts[ii * 2].key = &ids[isorted];
idacts[ii * 2 + 1].key = &ids[isorted];
}
idrq.cmp.compare = ebin_cmp;
idrq.cmp.arg = &tmpsb;
idrq.db = db;
idrq.actions = idacts;
idrq.num_actions = numdocs * 2;
idrq.reduce = by_id_reduce;
idrq.rereduce = by_id_rereduce;
idrq.fetch_callback = idfetch_update_cb;
idrq.db = db;
idrq.compacting = 0;
new_id_root = modify_btree(&idrq, db->header.by_id_root, &errcode);
error_pass(errcode);
while (fetcharg.valpos < numdocs) {
seqacts[fetcharg.actpos].type = ACTION_INSERT;
seqacts[fetcharg.actpos].value.data = &seqvals[fetcharg.valpos];
seqacts[fetcharg.actpos].key = &seqs[fetcharg.valpos];
fetcharg.valpos++;
fetcharg.actpos++;
}
//printf("Total seq actions: %d\n", fetcharg.actpos);
qsort(seqacts, fetcharg.actpos, sizeof(couchfile_modify_action),
seq_action_compare);
seqrq.cmp.compare = seq_cmp;
seqrq.cmp.arg = &tmpsb;
seqrq.actions = seqacts;
seqrq.num_actions = fetcharg.actpos;
seqrq.reduce = by_seq_reduce;
seqrq.rereduce = by_seq_rereduce;
seqrq.db = db;
seqrq.compacting = 0;
new_seq_root = modify_btree(&seqrq, db->header.by_seq_root, &errcode);
if (errcode != COUCHSTORE_SUCCESS) {
fatbuf_free(actbuf);
return errcode;
}
if (db->header.by_id_root != new_id_root) {
free(db->header.by_id_root);
db->header.by_id_root = new_id_root;
}
if (db->header.by_seq_root != new_seq_root) {
free(db->header.by_seq_root);
db->header.by_seq_root = new_seq_root;
}
cleanup:
free(sorted_ids);
fatbuf_free(actbuf);
return errcode;
}