/*
* __curds_remove --
* WT_CURSOR.remove method for the data-source cursor type.
*/
static int
__curds_remove(WT_CURSOR *cursor)
{
WT_CURSOR *source;
WT_DECL_RET;
WT_SESSION_IMPL *session;
source = ((WT_CURSOR_DATA_SOURCE *)cursor)->source;
CURSOR_UPDATE_API_CALL(cursor, session, remove, NULL);
WT_STAT_FAST_CONN_INCR(session, cursor_remove);
WT_STAT_FAST_DATA_INCR(session, cursor_remove);
WT_STAT_FAST_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size);
WT_ERR(__curds_txn_enter(session));
WT_ERR(__curds_key_set(cursor));
ret = __curds_cursor_resolve(cursor, source->remove(source));
err: __curds_txn_leave(session);
CURSOR_UPDATE_API_END(session, ret);
return (ret);
}
/*
* __curds_insert --
* WT_CURSOR.insert method for the data-source cursor type.
*/
static int
__curds_insert(WT_CURSOR *cursor)
{
WT_CURSOR *source;
WT_DECL_RET;
WT_SESSION_IMPL *session;
source = ((WT_CURSOR_DATA_SOURCE *)cursor)->source;
CURSOR_UPDATE_API_CALL(cursor, session, insert, NULL);
WT_ERR(__curds_txn_enter(session));
WT_STAT_FAST_CONN_INCR(session, cursor_insert);
WT_STAT_FAST_DATA_INCR(session, cursor_insert);
WT_STAT_FAST_DATA_INCRV(session,
cursor_insert_bytes, cursor->key.size + cursor->value.size);
if (!F_ISSET(cursor, WT_CURSTD_APPEND))
WT_ERR(__curds_key_set(cursor));
WT_ERR(__curds_value_set(cursor));
ret = __curds_cursor_resolve(cursor, source->insert(source));
err: __curds_txn_leave(session);
CURSOR_UPDATE_API_END(session, ret);
return (ret);
}
/*
* __wt_bt_read --
* Read a cookie referenced block into a buffer.
*/
int
__wt_bt_read(WT_SESSION_IMPL *session,
WT_ITEM *buf, const uint8_t *addr, size_t addr_size)
{
WT_BM *bm;
WT_BTREE *btree;
WT_DECL_ITEM(tmp);
WT_DECL_RET;
const WT_PAGE_HEADER *dsk;
size_t result_len;
btree = S2BT(session);
bm = btree->bm;
/*
* If anticipating a compressed block, read into a scratch buffer and
* decompress into the caller's buffer. Else, read directly into the
* caller's buffer.
*/
if (btree->compressor == NULL) {
WT_RET(bm->read(bm, session, buf, addr, addr_size));
dsk = buf->data;
} else {
WT_RET(__wt_scr_alloc(session, 0, &tmp));
WT_ERR(bm->read(bm, session, tmp, addr, addr_size));
dsk = tmp->data;
}
/*
* If the block is compressed, copy the skipped bytes of the original
* image into place, then decompress.
*/
if (F_ISSET(dsk, WT_PAGE_COMPRESSED)) {
if (btree->compressor == NULL ||
btree->compressor->decompress == NULL)
WT_ERR_MSG(session, WT_ERROR,
"read compressed block where no compression engine "
"configured");
/*
* We're allocating the exact number of bytes we're expecting
* from decompression.
*/
WT_ERR(__wt_buf_initsize(session, buf, dsk->mem_size));
/*
* Note the source length is NOT the number of compressed bytes,
* it's the length of the block we just read (minus the skipped
* bytes). We don't store the number of compressed bytes: some
* compression engines need that length stored externally, they
* don't have markers in the stream to signal the end of the
* compressed bytes. Those engines must store the compressed
* byte length somehow, see the snappy compression extension for
* an example.
*/
memcpy(buf->mem, tmp->data, WT_BLOCK_COMPRESS_SKIP);
ret = btree->compressor->decompress(
btree->compressor, &session->iface,
(uint8_t *)tmp->data + WT_BLOCK_COMPRESS_SKIP,
tmp->size - WT_BLOCK_COMPRESS_SKIP,
(uint8_t *)buf->mem + WT_BLOCK_COMPRESS_SKIP,
dsk->mem_size - WT_BLOCK_COMPRESS_SKIP, &result_len);
/*
* If checksums were turned off because we're depending on the
* decompression to fail on any corrupted data, we'll end up
* here after corruption happens. If we're salvaging the file,
* it's OK, otherwise it's really, really bad.
*/
if (ret != 0 ||
result_len != dsk->mem_size - WT_BLOCK_COMPRESS_SKIP)
WT_ERR(
F_ISSET(btree, WT_BTREE_VERIFY) ||
F_ISSET(session, WT_SESSION_SALVAGE_CORRUPT_OK) ?
WT_ERROR :
__wt_illegal_value(session, btree->dhandle->name));
} else
if (btree->compressor == NULL)
buf->size = dsk->mem_size;
else
/*
* We guessed wrong: there was a compressor, but this
* block was not compressed, and now the page is in the
* wrong buffer and the buffer may be of the wrong size.
* This should be rare, but happens with small blocks
* that aren't worth compressing.
*/
WT_ERR(__wt_buf_set(
session, buf, tmp->data, dsk->mem_size));
/* If the handle is a verify handle, verify the physical page. */
if (F_ISSET(btree, WT_BTREE_VERIFY)) {
if (tmp == NULL)
WT_ERR(__wt_scr_alloc(session, 0, &tmp));
WT_ERR(bm->addr_string(bm, session, tmp, addr, addr_size));
WT_ERR(__wt_verify_dsk(session, (const char *)tmp->data, buf));
}
WT_STAT_FAST_CONN_INCR(session, cache_read);
WT_STAT_FAST_DATA_INCR(session, cache_read);
if (F_ISSET(dsk, WT_PAGE_COMPRESSED))
WT_STAT_FAST_DATA_INCR(session, compress_read);
WT_STAT_FAST_CONN_INCRV(session, cache_bytes_read, dsk->mem_size);
WT_STAT_FAST_DATA_INCRV(session, cache_bytes_read, dsk->mem_size);
err: __wt_scr_free(session, &tmp);
return (ret);
}
/*
* __wt_bt_write --
* Write a buffer into a block, returning the block's addr/size and
* checksum.
*/
int
__wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf,
uint8_t *addr, size_t *addr_sizep, bool checkpoint, bool compressed)
{
WT_BM *bm;
WT_BTREE *btree;
WT_ITEM *ip;
WT_DECL_ITEM(tmp);
WT_DECL_RET;
WT_PAGE_HEADER *dsk;
size_t dst_len, len, result_len, size, src_len;
int compression_failed; /* Extension API, so not a bool. */
uint8_t *dst, *src;
bool data_cksum;
btree = S2BT(session);
bm = btree->bm;
/* Checkpoint calls are different than standard calls. */
WT_ASSERT(session,
(!checkpoint && addr != NULL && addr_sizep != NULL) ||
(checkpoint && addr == NULL && addr_sizep == NULL));
#ifdef HAVE_DIAGNOSTIC
/*
* We're passed a table's disk image. Decompress if necessary and
* verify the image. Always check the in-memory length for accuracy.
*/
dsk = buf->mem;
if (compressed) {
WT_ERR(__wt_scr_alloc(session, dsk->mem_size, &tmp));
memcpy(tmp->mem, buf->data, WT_BLOCK_COMPRESS_SKIP);
WT_ERR(btree->compressor->decompress(
btree->compressor, &session->iface,
(uint8_t *)buf->data + WT_BLOCK_COMPRESS_SKIP,
buf->size - WT_BLOCK_COMPRESS_SKIP,
(uint8_t *)tmp->data + WT_BLOCK_COMPRESS_SKIP,
tmp->memsize - WT_BLOCK_COMPRESS_SKIP,
&result_len));
WT_ASSERT(session,
dsk->mem_size == result_len + WT_BLOCK_COMPRESS_SKIP);
tmp->size = (uint32_t)result_len + WT_BLOCK_COMPRESS_SKIP;
ip = tmp;
} else {
WT_ASSERT(session, dsk->mem_size == buf->size);
ip = buf;
}
WT_ERR(__wt_verify_dsk(session, "[write-check]", ip));
__wt_scr_free(session, &tmp);
#endif
/*
* Optionally stream-compress the data, but don't compress blocks that
* are already as small as they're going to get.
*/
if (btree->compressor == NULL ||
btree->compressor->compress == NULL || compressed)
ip = buf;
else if (buf->size <= btree->allocsize) {
ip = buf;
WT_STAT_FAST_DATA_INCR(session, compress_write_too_small);
} else {
/* Skip the header bytes of the source data. */
src = (uint8_t *)buf->mem + WT_BLOCK_COMPRESS_SKIP;
src_len = buf->size - WT_BLOCK_COMPRESS_SKIP;
/*
* Compute the size needed for the destination buffer. We only
* allocate enough memory for a copy of the original by default,
* if any compressed version is bigger than the original, we
* won't use it. However, some compression engines (snappy is
* one example), may need more memory because they don't stop
* just because there's no more memory into which to compress.
*/
if (btree->compressor->pre_size == NULL)
len = src_len;
else
WT_ERR(btree->compressor->pre_size(btree->compressor,
&session->iface, src, src_len, &len));
size = len + WT_BLOCK_COMPRESS_SKIP;
WT_ERR(bm->write_size(bm, session, &size));
WT_ERR(__wt_scr_alloc(session, size, &tmp));
/* Skip the header bytes of the destination data. */
dst = (uint8_t *)tmp->mem + WT_BLOCK_COMPRESS_SKIP;
dst_len = len;
compression_failed = 0;
WT_ERR(btree->compressor->compress(btree->compressor,
&session->iface,
src, src_len,
dst, dst_len,
&result_len, &compression_failed));
result_len += WT_BLOCK_COMPRESS_SKIP;
/*
* If compression fails, or doesn't gain us at least one unit of
* allocation, fallback to the original version. This isn't
* unexpected: if compression doesn't work for some chunk of
* data for some reason (noting likely additional format/header
* information which compressed output requires), it just means
* the uncompressed version is as good as it gets, and that's
* what we use.
*/
if (compression_failed ||
buf->size / btree->allocsize <=
result_len / btree->allocsize) {
ip = buf;
WT_STAT_FAST_DATA_INCR(session, compress_write_fail);
} else {
compressed = true;
WT_STAT_FAST_DATA_INCR(session, compress_write);
/*
* Copy in the skipped header bytes, set the final data
* size.
*/
memcpy(tmp->mem, buf->mem, WT_BLOCK_COMPRESS_SKIP);
tmp->size = result_len;
ip = tmp;
}
}
dsk = ip->mem;
/* If the buffer is compressed, set the flag. */
if (compressed)
F_SET(dsk, WT_PAGE_COMPRESSED);
/*
* We increment the block's write generation so it's easy to identify
* newer versions of blocks during salvage. (It's common in WiredTiger,
* at least for the default block manager, for multiple blocks to be
* internally consistent with identical first and last keys, so we need
* a way to know the most recent state of the block. We could check
* which leaf is referenced by a valid internal page, but that implies
* salvaging internal pages, which I don't want to do, and it's not
* as good anyway, because the internal page may not have been written
* after the leaf page was updated. So, write generations it is.
*
* Nothing is locked at this point but two versions of a page with the
* same generation is pretty unlikely, and if we did, they're going to
* be roughly identical for the purposes of salvage, anyway.
*/
dsk->write_gen = ++btree->write_gen;
/*
* Checksum the data if the buffer isn't compressed or checksums are
* configured.
*/
switch (btree->checksum) {
case CKSUM_ON:
data_cksum = true;
break;
case CKSUM_OFF:
data_cksum = false;
break;
case CKSUM_UNCOMPRESSED:
default:
data_cksum = !compressed;
break;
}
/* Call the block manager to write the block. */
WT_ERR(checkpoint ?
bm->checkpoint(bm, session, ip, btree->ckpt, data_cksum) :
bm->write(bm, session, ip, addr, addr_sizep, data_cksum));
WT_STAT_FAST_CONN_INCR(session, cache_write);
WT_STAT_FAST_DATA_INCR(session, cache_write);
WT_STAT_FAST_CONN_INCRV(session, cache_bytes_write, dsk->mem_size);
WT_STAT_FAST_DATA_INCRV(session, cache_bytes_write, dsk->mem_size);
err: __wt_scr_free(session, &tmp);
return (ret);
}
/*
* __wt_btcur_update --
* Update a record in the tree.
*/
int
__wt_btcur_update(WT_CURSOR_BTREE *cbt)
{
WT_BTREE *btree;
WT_CURSOR *cursor;
WT_DECL_RET;
WT_SESSION_IMPL *session;
btree = cbt->btree;
cursor = &cbt->iface;
session = (WT_SESSION_IMPL *)cursor->session;
WT_STAT_FAST_CONN_INCR(session, cursor_update);
WT_STAT_FAST_DATA_INCR(session, cursor_update);
WT_STAT_FAST_DATA_INCRV(
session, cursor_update_bytes, cursor->value.size);
if (btree->type == BTREE_ROW)
WT_RET(__cursor_size_chk(session, &cursor->key));
WT_RET(__cursor_size_chk(session, &cursor->value));
/*
* The tree is no longer empty: eviction should pay attention to it,
* and it's no longer possible to bulk-load into it.
*/
if (btree->bulk_load_ok) {
btree->bulk_load_ok = 0;
__wt_btree_evictable(session, 1);
}
retry: WT_RET(__cursor_func_init(cbt, 1));
switch (btree->type) {
case BTREE_COL_FIX:
case BTREE_COL_VAR:
WT_ERR(__cursor_col_search(session, cbt));
/*
* If not overwriting, fail if the key doesn't exist. Update
* the record if it exists. Creating a record past the end of
* the tree in a fixed-length column-store implicitly fills the
* gap with empty records. Update the record in that case, the
* record exists.
*/
if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) &&
(cbt->compare != 0 || !__cursor_valid(cbt, NULL)) &&
!__cursor_fix_implicit(btree, cbt))
WT_ERR(WT_NOTFOUND);
ret = __cursor_col_modify(session, cbt, 0);
break;
case BTREE_ROW:
WT_ERR(__cursor_row_search(session, cbt, 1));
/*
* If not overwriting, fail if the key does not exist.
*/
if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) &&
(cbt->compare != 0 || !__cursor_valid(cbt, NULL)))
WT_ERR(WT_NOTFOUND);
ret = __cursor_row_modify(session, cbt, 0);
break;
WT_ILLEGAL_VALUE_ERR(session);
}
err: if (ret == WT_RESTART)
goto retry;
/*
* If successful, point the cursor at internal copies of the data. We
* could shuffle memory in the cursor so the key/value pair are in local
* buffer memory, but that's a data copy. We don't want to do another
* search (and we might get a different update structure if we race).
* To make this work, we add a field to the btree cursor to pass back a
* pointer to the modify function's allocated update structure.
*/
if (ret == 0)
WT_TRET(__wt_kv_return(session, cbt, cbt->modify_update));
if (ret != 0)
WT_TRET(__cursor_reset(cbt));
return (ret);
}
/*
* __wt_btcur_remove --
* Remove a record from the tree.
*/
int
__wt_btcur_remove(WT_CURSOR_BTREE *cbt)
{
WT_BTREE *btree;
WT_CURSOR *cursor;
WT_DECL_RET;
WT_SESSION_IMPL *session;
btree = cbt->btree;
cursor = &cbt->iface;
session = (WT_SESSION_IMPL *)cursor->session;
WT_STAT_FAST_CONN_INCR(session, cursor_remove);
WT_STAT_FAST_DATA_INCR(session, cursor_remove);
WT_STAT_FAST_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size);
if (btree->type == BTREE_ROW)
WT_RET(__cursor_size_chk(session, &cursor->key));
retry: WT_RET(__cursor_func_init(cbt, 1));
switch (btree->type) {
case BTREE_COL_FIX:
case BTREE_COL_VAR:
WT_ERR(__cursor_col_search(session, cbt));
/* Remove the record if it exists. */
if (cbt->compare != 0 || !__cursor_valid(cbt, NULL)) {
if (!__cursor_fix_implicit(btree, cbt))
WT_ERR(WT_NOTFOUND);
/*
* Creating a record past the end of the tree in a
* fixed-length column-store implicitly fills the
* gap with empty records. Return success in that
* case, the record was deleted successfully.
*
* Correct the btree cursor's location: the search
* will have pointed us at the previous/next item,
* and that's not correct.
*/
cbt->recno = cursor->recno;
} else
ret = __cursor_col_modify(session, cbt, 1);
break;
case BTREE_ROW:
/* Remove the record if it exists. */
WT_ERR(__cursor_row_search(session, cbt, 0));
if (cbt->compare != 0 || !__cursor_valid(cbt, NULL))
WT_ERR(WT_NOTFOUND);
ret = __cursor_row_modify(session, cbt, 1);
break;
WT_ILLEGAL_VALUE_ERR(session);
}
err: if (ret == WT_RESTART)
goto retry;
/*
* If the cursor is configured to overwrite and the record is not
* found, that is exactly what we want.
*/
if (F_ISSET(cursor, WT_CURSTD_OVERWRITE) && ret == WT_NOTFOUND)
ret = 0;
if (ret != 0)
WT_TRET(__cursor_reset(cbt));
return (ret);
}
/*
* __wt_btcur_insert --
* Insert a record into the tree.
*/
int
__wt_btcur_insert(WT_CURSOR_BTREE *cbt)
{
WT_BTREE *btree;
WT_CURSOR *cursor;
WT_DECL_RET;
WT_SESSION_IMPL *session;
btree = cbt->btree;
cursor = &cbt->iface;
session = (WT_SESSION_IMPL *)cursor->session;
WT_STAT_FAST_CONN_INCR(session, cursor_insert);
WT_STAT_FAST_DATA_INCR(session, cursor_insert);
WT_STAT_FAST_DATA_INCRV(session,
cursor_insert_bytes, cursor->key.size + cursor->value.size);
if (btree->type == BTREE_ROW)
WT_RET(__cursor_size_chk(session, &cursor->key));
WT_RET(__cursor_size_chk(session, &cursor->value));
/*
* The tree is no longer empty: eviction should pay attention to it,
* and it's no longer possible to bulk-load into it.
*/
if (btree->bulk_load_ok) {
btree->bulk_load_ok = 0;
__wt_btree_evictable(session, 1);
}
retry: WT_RET(__cursor_func_init(cbt, 1));
switch (btree->type) {
case BTREE_COL_FIX:
case BTREE_COL_VAR:
/*
* If WT_CURSTD_APPEND is set, insert a new record (ignoring
* the application's record number). First we search for the
* maximum possible record number so the search ends on the
* last page. The real record number is assigned by the
* serialized append operation.
*/
if (F_ISSET(cursor, WT_CURSTD_APPEND))
cbt->iface.recno = UINT64_MAX;
WT_ERR(__cursor_col_search(session, cbt));
if (F_ISSET(cursor, WT_CURSTD_APPEND))
cbt->iface.recno = 0;
/*
* If not overwriting, fail if the key exists. Creating a
* record past the end of the tree in a fixed-length
* column-store implicitly fills the gap with empty records.
* Fail in that case, the record exists.
*/
if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) &&
((cbt->compare == 0 && __cursor_valid(cbt, NULL)) ||
(cbt->compare != 0 && __cursor_fix_implicit(btree, cbt))))
WT_ERR(WT_DUPLICATE_KEY);
WT_ERR(__cursor_col_modify(session, cbt, 0));
if (F_ISSET(cursor, WT_CURSTD_APPEND))
cbt->iface.recno = cbt->recno;
break;
case BTREE_ROW:
WT_ERR(__cursor_row_search(session, cbt, 1));
/*
* If not overwriting, fail if the key exists, else insert the
* key/value pair.
*/
if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) &&
cbt->compare == 0 && __cursor_valid(cbt, NULL))
WT_ERR(WT_DUPLICATE_KEY);
ret = __cursor_row_modify(session, cbt, 0);
break;
WT_ILLEGAL_VALUE_ERR(session);
}
err: if (ret == WT_RESTART)
goto retry;
/* Insert doesn't maintain a position across calls, clear resources. */
if (ret == 0)
WT_TRET(__curfile_leave(cbt));
if (ret != 0)
WT_TRET(__cursor_reset(cbt));
return (ret);
}
/*
* __wt_bt_read --
* Read a cookie referenced block into a buffer.
*/
int
__wt_bt_read(WT_SESSION_IMPL *session,
WT_ITEM *buf, const uint8_t *addr, size_t addr_size)
{
WT_BM *bm;
WT_BTREE *btree;
WT_DECL_ITEM(etmp);
WT_DECL_ITEM(tmp);
WT_DECL_RET;
WT_ENCRYPTOR *encryptor;
WT_ITEM *ip;
const WT_PAGE_HEADER *dsk;
const char *fail_msg;
size_t result_len;
btree = S2BT(session);
bm = btree->bm;
fail_msg = NULL; /* -Wuninitialized */
/*
* If anticipating a compressed or encrypted block, read into a scratch
* buffer and decompress into the caller's buffer. Else, read directly
* into the caller's buffer.
*/
if (btree->compressor == NULL && btree->kencryptor == NULL) {
WT_RET(bm->read(bm, session, buf, addr, addr_size));
dsk = buf->data;
ip = NULL;
} else {
WT_RET(__wt_scr_alloc(session, 0, &tmp));
WT_ERR(bm->read(bm, session, tmp, addr, addr_size));
dsk = tmp->data;
ip = tmp;
}
/*
* If the block is encrypted, copy the skipped bytes of the original
* image into place, then decrypt.
*/
if (F_ISSET(dsk, WT_PAGE_ENCRYPTED)) {
if (btree->kencryptor == NULL ||
(encryptor = btree->kencryptor->encryptor) == NULL ||
encryptor->decrypt == NULL) {
fail_msg =
"encrypted block in file for which no encryption "
"configured";
goto corrupt;
}
WT_ERR(__wt_scr_alloc(session, 0, &etmp));
if ((ret = __wt_decrypt(session,
encryptor, WT_BLOCK_ENCRYPT_SKIP, ip, etmp)) != 0) {
fail_msg = "block decryption failed";
goto corrupt;
}
ip = etmp;
dsk = ip->data;
} else if (btree->kencryptor != NULL) {
fail_msg =
"unencrypted block in file for which encryption configured";
goto corrupt;
}
if (F_ISSET(dsk, WT_PAGE_COMPRESSED)) {
if (btree->compressor == NULL ||
btree->compressor->decompress == NULL) {
fail_msg =
"compressed block in file for which no compression "
"configured";
goto corrupt;
}
/*
* Size the buffer based on the in-memory bytes we're expecting
* from decompression.
*/
WT_ERR(__wt_buf_initsize(session, buf, dsk->mem_size));
/*
* Note the source length is NOT the number of compressed bytes,
* it's the length of the block we just read (minus the skipped
* bytes). We don't store the number of compressed bytes: some
* compression engines need that length stored externally, they
* don't have markers in the stream to signal the end of the
* compressed bytes. Those engines must store the compressed
* byte length somehow, see the snappy compression extension for
* an example.
*/
memcpy(buf->mem, ip->data, WT_BLOCK_COMPRESS_SKIP);
ret = btree->compressor->decompress(
btree->compressor, &session->iface,
(uint8_t *)ip->data + WT_BLOCK_COMPRESS_SKIP,
tmp->size - WT_BLOCK_COMPRESS_SKIP,
(uint8_t *)buf->mem + WT_BLOCK_COMPRESS_SKIP,
dsk->mem_size - WT_BLOCK_COMPRESS_SKIP, &result_len);
/*
* If checksums were turned off because we're depending on the
* decompression to fail on any corrupted data, we'll end up
* here after corruption happens. If we're salvaging the file,
* it's OK, otherwise it's really, really bad.
*/
if (ret != 0 ||
result_len != dsk->mem_size - WT_BLOCK_COMPRESS_SKIP) {
fail_msg = "block decryption failed";
goto corrupt;
}
} else
/*
* If we uncompressed above, the page is in the correct buffer.
* If we get here the data may be in the wrong buffer and the
* buffer may be the wrong size. If needed, get the page
* into the destination buffer.
*/
if (ip != NULL)
WT_ERR(__wt_buf_set(
session, buf, ip->data, dsk->mem_size));
/* If the handle is a verify handle, verify the physical page. */
if (F_ISSET(btree, WT_BTREE_VERIFY)) {
if (tmp == NULL)
WT_ERR(__wt_scr_alloc(session, 0, &tmp));
WT_ERR(bm->addr_string(bm, session, tmp, addr, addr_size));
WT_ERR(__wt_verify_dsk(session, tmp->data, buf));
}
WT_STAT_FAST_CONN_INCR(session, cache_read);
WT_STAT_FAST_DATA_INCR(session, cache_read);
if (F_ISSET(dsk, WT_PAGE_COMPRESSED))
WT_STAT_FAST_DATA_INCR(session, compress_read);
WT_STAT_FAST_CONN_INCRV(session, cache_bytes_read, dsk->mem_size);
WT_STAT_FAST_DATA_INCRV(session, cache_bytes_read, dsk->mem_size);
if (0) {
corrupt: if (ret == 0)
ret = WT_ERROR;
if (!F_ISSET(btree, WT_BTREE_VERIFY) &&
!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) {
__wt_err(session, ret, "%s", fail_msg);
ret = __wt_illegal_value(session, btree->dhandle->name);
}
}
err: __wt_scr_free(session, &tmp);
__wt_scr_free(session, &etmp);
return (ret);
}
/*
* __wt_merge_tree --
* Attempt to collapse a stack of split-merge pages in memory into a
* shallow tree. If enough keys are found, create a real internal node
* that can be evicted (and, if necessary, split further).
*
* This code is designed to deal with workloads that otherwise create
* arbitrarily deep (and slow) trees in memory.
*/
int
__wt_merge_tree(WT_SESSION_IMPL *session, WT_PAGE *top)
{
WT_DECL_RET;
WT_PAGE *lchild, *newtop, *rchild;
WT_REF *newref;
WT_VISIT_STATE visit_state;
uint32_t refcnt, split;
int promote;
u_int levels;
uint8_t page_type;
WT_CLEAR(visit_state);
visit_state.session = session;
lchild = newtop = rchild = NULL;
page_type = top->type;
WT_ASSERT(session, __wt_btree_mergeable(top));
WT_ASSERT(session, top->ref->state == WT_REF_LOCKED);
/*
* Walk the subtree, count the references at the bottom level and
* calculate the maximum depth.
*/
WT_RET(__merge_walk(session, top, 1, __merge_count, &visit_state));
/* If there aren't enough useful levels, give up. */
if (visit_state.maxdepth < WT_MERGE_STACK_MIN)
return (EBUSY);
/* Pages cannot grow larger than 2**32, but that should never happen. */
if (visit_state.refcnt > UINT32_MAX)
return (ENOMEM);
/*
* Now we either collapse the internal pages into one split-merge page,
* or if there are "enough" keys, we split into two equal internal
* pages, each of which can be evicted independently.
*
* We set a flag (WT_PM_REC_SPLIT_MERGE) on the created page if it
* isn't big enough to justify the cost of evicting it. If splits
* continue, it will be merged again until it gets over this limit.
*/
promote = 0;
refcnt = (uint32_t)visit_state.refcnt;
if (refcnt >= WT_MERGE_FULL_PAGE && visit_state.seen_live) {
/*
* In the normal case where there are live children spread
* through the subtree, create two child pages.
*
* Handle the case where the live children are all near the
* beginning / end specially: put the last live child into the
* top-level page, to avoid getting much deeper during
* append-only workloads.
*
* Set SPLIT_MERGE on the internal pages if there are any live
* children: they can't be evicted, so there is no point
* permanently deepening the tree.
*/
if (visit_state.last_live <= refcnt / 10)
split = 1;
else if (visit_state.first_live >= (9 * refcnt) / 10)
split = refcnt - 1;
else
split = (refcnt + 1) / 2;
/* Only promote if we can create a real page. */
if (split == 1 || split == refcnt - 1)
promote = 1;
else if (split >= WT_MERGE_FULL_PAGE &&
visit_state.first_live >= split)
promote = 1;
else if (refcnt - split >= WT_MERGE_FULL_PAGE &&
visit_state.last_live < split)
promote = 1;
}
if (promote) {
/* Create a new top-level split-merge page with two entries. */
WT_ERR(__wt_btree_new_modified_page(
session, page_type, 2, 1, &newtop));
visit_state.split = split;
/* Left split. */
if (split == 1)
visit_state.first = newtop;
else {
WT_ERR(__wt_btree_new_modified_page(
session, page_type, split,
split < WT_MERGE_FULL_PAGE, &lchild));
visit_state.first = lchild;
}
/* Right split. */
if (split == refcnt - 1) {
visit_state.second = newtop;
visit_state.second_ref = &newtop->u.intl.t[1];
} else {
WT_ERR(__wt_btree_new_modified_page(
session, page_type, refcnt - split,
refcnt - split < WT_MERGE_FULL_PAGE,
&rchild));
visit_state.second = rchild;
visit_state.second_ref =
&visit_state.second->u.intl.t[0];
}
} else {
/*
* Create a new split-merge page for small merges. When we do
* a big enough merge, we create a real page at the top and
* don't consider it as a merge candidate again. Over time
* with an insert workload the tree will grow deeper, but
* that's inevitable, and this keeps individual merges small.
*/
WT_ERR(__wt_btree_new_modified_page(
session, page_type, refcnt,
refcnt < WT_MERGE_FULL_PAGE,
&newtop));
visit_state.first = newtop;
}
/*
* Copy the references into the new tree, but don't update anything in
* the locked tree in case there is an error and we need to back out.
* We do this in a separate pass so that we can figure out the key for
* the split point: that allocates memory and so it could still fail.
*/
visit_state.page = visit_state.first;
visit_state.ref = visit_state.page->u.intl.t;
visit_state.refcnt = 0;
WT_ERR(__merge_walk(session, top, 0, __merge_copy_ref, &visit_state));
if (promote) {
/* Promote keys into the top-level page. */
if (lchild != NULL) {
newref = &newtop->u.intl.t[0];
WT_LINK_PAGE(newtop, newref, lchild);
newref->state = WT_REF_MEM;
WT_ERR(__merge_promote_key(session, newref));
}
if (rchild != NULL) {
newref = &newtop->u.intl.t[1];
WT_LINK_PAGE(newtop, newref, rchild);
newref->state = WT_REF_MEM;
WT_ERR(__merge_promote_key(session, newref));
}
}
/*
* We have copied everything into place and allocated all of the memory
* we need. Now link all pages into the new tree and unlock them.
*
* The only way this could fail is if a reference state has been
* changed by another thread since they were locked. Panic in that
* case: that should never happen.
*/
visit_state.page = visit_state.first;
visit_state.ref = visit_state.page->u.intl.t;
visit_state.refcnt = 0;
ret = __merge_walk(session, top, 0, __merge_switch_page, &visit_state);
if (ret != 0)
WT_ERR(__wt_illegal_value(session, "__wt_merge_tree"));
newtop->u.intl.recno = top->u.intl.recno;
newtop->parent = top->parent;
newtop->ref = top->ref;
#ifdef HAVE_DIAGNOSTIC
/*
* Before swapping in the new tree, walk the pages we are discarding,
* check that everything looks right.
*/
__merge_check_discard(session, top);
#endif
/*
* Set up the new top-level page as a split so that it will be swapped
* into place by our caller.
*/
top->modify->flags = WT_PM_REC_SPLIT;
top->modify->u.split = newtop;
WT_VERBOSE_ERR(session, evict,
"Successfully %s %" PRIu32
" split-merge pages containing %" PRIu32 " keys\n",
promote ? "promoted" : "merged", visit_state.maxdepth, refcnt);
/* Evict new child pages as soon as possible. */
if (lchild != NULL && !F_ISSET(lchild->modify, WT_PM_REC_SPLIT_MERGE))
lchild->read_gen = WT_READ_GEN_OLDEST;
if (rchild != NULL && !F_ISSET(rchild->modify, WT_PM_REC_SPLIT_MERGE))
rchild->read_gen = WT_READ_GEN_OLDEST;
/* Update statistics. */
WT_STAT_FAST_CONN_INCR(session, cache_eviction_merge);
WT_STAT_FAST_DATA_INCR(session, cache_eviction_merge);
/* How many levels did we remove? */
levels = visit_state.maxdepth - (promote ? 2 : 1);
WT_STAT_FAST_CONN_INCRV(session, cache_eviction_merge_levels, levels);
WT_STAT_FAST_DATA_INCRV(session, cache_eviction_merge_levels, levels);
return (0);
err: WT_VERBOSE_TRET(session, evict,
"Failed to merge %" PRIu32
" split-merge pages containing %" PRIu32 " keys\n",
visit_state.maxdepth, refcnt);
WT_STAT_FAST_CONN_INCR(session, cache_eviction_merge_fail);
WT_STAT_FAST_DATA_INCR(session, cache_eviction_merge_fail);
if (newtop != NULL)
__wt_page_out(session, &newtop);
if (lchild != NULL)
__wt_page_out(session, &lchild);
if (rchild != NULL)
__wt_page_out(session, &rchild);
return (ret);
}
