static couchstore_error_t modify_node(couchfile_modify_request *rq,
node_pointer *nptr,
int start, int end,
couchfile_modify_result *dst)
{
char *nodebuf = NULL; // FYI, nodebuf is a malloced block, not in the arena
int bufpos = 1;
int nodebuflen = 0;
int errcode = 0;
couchfile_modify_result *local_result = NULL;
if (start == end) {
return 0;
}
if (nptr) {
if ((nodebuflen = pread_compressed(rq->db, nptr->pointer, (char **) &nodebuf)) < 0) {
error_pass(COUCHSTORE_ERROR_READ);
}
}
local_result = make_modres(dst->arena, rq);
error_unless(local_result, COUCHSTORE_ERROR_ALLOC_FAIL);
if (nptr == NULL || nodebuf[0] == 1) { //KV Node
local_result->node_type = KV_NODE;
while (bufpos < nodebuflen) {
sized_buf cmp_key, val_buf;
bufpos += read_kv(nodebuf + bufpos, &cmp_key, &val_buf);
int advance = 0;
while (!advance && start < end) {
advance = 1;
int cmp_val = rq->cmp.compare(&cmp_key, rq->actions[start].key);
if (cmp_val < 0) { //Key less than action key
mr_push_item(&cmp_key, &val_buf, local_result);
} else if (cmp_val > 0) { //Key greater than action key
switch (rq->actions[start].type) {
case ACTION_INSERT:
local_result->modified = 1;
mr_push_item(rq->actions[start].key, rq->actions[start].value.data, local_result);
break;
case ACTION_REMOVE:
local_result->modified = 1;
break;
case ACTION_FETCH:
if (rq->fetch_callback) {
//not found
(*rq->fetch_callback)(rq, rq->actions[start].key, NULL, rq->actions[start].value.arg);
}
}
start++;
//Do next action on same item in the node, as our action was
//not >= it.
advance = 0;
} else if (cmp_val == 0) { //Node key is equal to action key
switch (rq->actions[start].type) {
case ACTION_INSERT:
local_result->modified = 1;
mr_push_item(rq->actions[start].key, rq->actions[start].value.data, local_result);
break;
case ACTION_REMOVE:
local_result->modified = 1;
break;
case ACTION_FETCH:
if (rq->fetch_callback) {
(*rq->fetch_callback)(rq, rq->actions[start].key, &val_buf, rq->actions[start].value.arg);
}
//Do next action on same item in the node, as our action was a fetch
//and there may be an equivalent insert or remove
//following.
advance = 0;
}
start++;
}
}
if (start == end && !advance) {
//If we've exhausted actions then just keep this key
mr_push_item(&cmp_key, &val_buf, local_result);
}
}
while (start < end) {
//We're at the end of a leaf node.
switch (rq->actions[start].type) {
case ACTION_INSERT:
local_result->modified = 1;
mr_push_item(rq->actions[start].key, rq->actions[start].value.data, local_result);
break;
case ACTION_REMOVE:
local_result->modified = 1;
break;
case ACTION_FETCH:
if (rq->fetch_callback) {
//not found
(*rq->fetch_callback)(rq, rq->actions[start].key, NULL, rq->actions[start].value.arg);
}
break;
}
start++;
}
} else if (nodebuf[0] == 0) { //KP Node
local_result->node_type = KP_NODE;
while (bufpos < nodebuflen && start < end) {
sized_buf cmp_key, val_buf;
bufpos += read_kv(nodebuf + bufpos, &cmp_key, &val_buf);
int cmp_val = rq->cmp.compare(&cmp_key, rq->actions[start].key);
if (bufpos == nodebuflen) {
//We're at the last item in the kpnode, must apply all our
//actions here.
node_pointer *desc = read_pointer(dst->arena, &cmp_key, val_buf.buf);
if (!desc) {
errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
goto cleanup;
}
errcode = modify_node(rq, desc, start, end, local_result);
if (errcode != COUCHSTORE_SUCCESS) {
goto cleanup;
}
break;
}
if (cmp_val < 0) {
//Key in node item less than action item and not at end
//position, so just add it and continue.
node_pointer *add = read_pointer(dst->arena, &cmp_key, val_buf.buf);
if (!add) {
errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
goto cleanup;
}
errcode = mr_push_pointerinfo(add, local_result);
if (errcode != COUCHSTORE_SUCCESS) {
goto cleanup;
}
} else if (cmp_val >= 0) {
//Found a key in the node greater than the one in the current
//action. Descend into the pointed node with as many actions as
//are less than the key here.
int range_end = start;
while (range_end < end &&
rq->cmp.compare(rq->actions[range_end].key, &cmp_key) <= 0) {
range_end++;
}
node_pointer *desc = read_pointer(dst->arena, &cmp_key, val_buf.buf);
if (!desc) {
errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
goto cleanup;
}
errcode = modify_node(rq, desc, start, range_end, local_result);
start = range_end;
if (errcode != COUCHSTORE_SUCCESS) {
goto cleanup;
}
}
}
while (bufpos < nodebuflen) {
sized_buf cmp_key, val_buf;
bufpos += read_kv(nodebuf + bufpos, &cmp_key, &val_buf);
node_pointer *add = read_pointer(dst->arena, &cmp_key, val_buf.buf);
if (!add) {
errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
goto cleanup;
}
errcode = mr_push_pointerinfo(add, local_result);
if (errcode != COUCHSTORE_SUCCESS) {
goto cleanup;
}
}
} else {
errcode = COUCHSTORE_ERROR_CORRUPT;
goto cleanup;
}
//If we've done modifications, write out the last leaf node.
error_pass(flush_mr(local_result));
if (!local_result->modified && nptr != NULL) {
//If we didn't do anything, give back the pointer to the original
mr_push_pointerinfo(nptr, dst);
} else {
//Otherwise, give back the pointers to the nodes we've created.
dst->modified = 1;
error_pass(mr_move_pointers(local_result, dst));
}
cleanup:
if (nodebuf) {
free(nodebuf);
}
return errcode;
}
couchstore_error_t TreeWriterWrite(TreeWriter* writer,
tree_file* treefile,
node_pointer** out_root)
{
couchstore_error_t errcode = COUCHSTORE_SUCCESS;
arena* transient_arena = new_arena(0);
arena* persistent_arena = new_arena(0);
error_unless(transient_arena && persistent_arena, COUCHSTORE_ERROR_ALLOC_FAIL);
rewind(writer->file);
// Create the structure to write the tree to the db:
compare_info idcmp;
sized_buf tmp;
idcmp.compare = writer->key_compare;
idcmp.arg = &tmp;
couchfile_modify_result* target_mr = new_btree_modres(persistent_arena,
transient_arena,
treefile, &idcmp,
writer->reduce,
writer->rereduce,
DB_CHUNK_THRESHOLD,
DB_CHUNK_THRESHOLD);
if(target_mr == NULL) {
error_pass(COUCHSTORE_ERROR_ALLOC_FAIL);
}
// Read all the key/value pairs from the file and add them to the tree:
uint16_t klen;
uint32_t vlen;
sized_buf k, v;
while(1) {
if(fread(&klen, sizeof(klen), 1, writer->file) != 1) {
break;
}
if(fread(&vlen, sizeof(vlen), 1, writer->file) != 1) {
break;
}
k.size = ntohs(klen);
k.buf = arena_alloc(transient_arena, k.size);
v.size = ntohl(vlen);
v.buf = arena_alloc(transient_arena, v.size);
if(fread(k.buf, k.size, 1, writer->file) != 1) {
error_pass(COUCHSTORE_ERROR_READ);
}
if(fread(v.buf, v.size, 1, writer->file) != 1) {
error_pass(COUCHSTORE_ERROR_READ);
}
//printf("K: '%.*s'\n", k.size, k.buf);
mr_push_item(&k, &v, target_mr);
if(target_mr->count == 0) {
/* No items queued, we must have just flushed. We can safely rewind the transient arena. */
arena_free_all(transient_arena);
}
}
// Check for file error:
int readerr = ferror(writer->file);
if(readerr != 0 && readerr != EOF) {
error_pass(COUCHSTORE_ERROR_READ);
}
// Finish up the tree:
*out_root = complete_new_btree(target_mr, &errcode);
cleanup:
delete_arena(transient_arena);
delete_arena(persistent_arena);
return errcode;
}
static couchstore_error_t build_btree(const char *source_file,
tree_file *dest_file,
compare_info *cmp,
reduce_fn reduce_fun,
reduce_fn rereduce_fun,
void *reduce_ctx,
node_pointer **out_root)
{
couchstore_error_t ret = COUCHSTORE_SUCCESS;
arena *transient_arena = new_arena(0);
arena *persistent_arena = new_arena(0);
couchfile_modify_result *mr;
FILE *f = NULL;
if (transient_arena == NULL || persistent_arena == NULL) {
ret = COUCHSTORE_ERROR_ALLOC_FAIL;
goto out;
}
mr = new_btree_modres(persistent_arena,
transient_arena,
dest_file,
cmp,
reduce_fun,
rereduce_fun,
reduce_ctx,
VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
if (mr == NULL) {
ret = COUCHSTORE_ERROR_ALLOC_FAIL;
goto out;
}
f = fopen(source_file, "rb");
if (f == NULL) {
ret = COUCHSTORE_ERROR_OPEN_FILE;
goto out;
}
while (1) {
sized_buf k, v;
int read_ret;
read_ret = read_record(f, transient_arena, &k, &v);
if (read_ret == 0) {
break;
} else if (read_ret < 0) {
ret = (couchstore_error_t) read_ret;
goto out;
}
ret = mr_push_item(&k, &v, mr);
if (ret != COUCHSTORE_SUCCESS) {
goto out;
}
if (mr->count == 0) {
arena_free_all(transient_arena);
}
}
*out_root = complete_new_btree(mr, &ret);
if (ret != COUCHSTORE_SUCCESS) {
goto out;
}
/* Don't care about success/failure. Erlang side will eventually delete it. */
remove(source_file);
out:
if (f != NULL) {
fclose(f);
}
if (transient_arena != NULL) {
delete_arena(transient_arena);
}
if (persistent_arena != NULL) {
delete_arena(persistent_arena);
}
return ret;
}