/**********************************************************************
Takes a block out of the LRU list and page hash table and sets the block
state to BUF_BLOCK_REMOVE_HASH. */
static
void
buf_LRU_block_remove_hashed_page(
/*=============================*/
buf_block_t* block) /* in: block, must contain a file page and
be in a state where it can be freed; there
may or may not be a hash index to the page */
{
ut_ad(mutex_own(&(buf_pool->mutex)));
ut_ad(mutex_own(&block->mutex));
ut_ad(block);
ut_a(block->state == BUF_BLOCK_FILE_PAGE);
ut_a(block->io_fix == 0);
ut_a(block->buf_fix_count == 0);
ut_a(ut_dulint_cmp(block->oldest_modification, ut_dulint_zero) == 0);
buf_LRU_remove_block(block);
buf_pool->freed_page_clock += 1;
/* Note that if AWE is enabled the block may not have a frame at all */
buf_block_modify_clock_inc(block);
if (block != buf_page_hash_get(block->space, block->offset)) {
fprintf(stderr,
"InnoDB: Error: page %lu %lu not found"
" in the hash table\n",
(ulong) block->space,
(ulong) block->offset);
if (buf_page_hash_get(block->space, block->offset)) {
fprintf(stderr,
"InnoDB: In hash table we find block"
" %p of %lu %lu which is not %p\n",
(void*) buf_page_hash_get
(block->space, block->offset),
(ulong) buf_page_hash_get
(block->space, block->offset)->space,
(ulong) buf_page_hash_get
(block->space, block->offset)->offset,
(void*) block);
}
#ifdef UNIV_DEBUG
buf_print();
buf_LRU_print();
buf_validate();
buf_LRU_validate();
#endif
ut_a(0);
}
HASH_DELETE(buf_block_t, hash, buf_pool->page_hash,
buf_page_address_fold(block->space, block->offset),
block);
UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE);
block->state = BUF_BLOCK_REMOVE_HASH;
}
ibool
buf_LRU_free_block(
/*===============*/
/* out: TRUE if freed */
buf_block_t* block) /* in/out: block to be freed */
{
if (!buf_flush_ready_for_replace(block)) {
return(FALSE);
}
#ifdef UNIV_DEBUG
if (buf_debug_prints) {
fprintf(stderr,
"Putting space %lu page %lu"
" to free list\n",
(ulong) block->space,
(ulong) block->offset);
}
#endif /* UNIV_DEBUG */
buf_LRU_block_remove_hashed_page(block);
mutex_exit(&(buf_pool->mutex));
mutex_exit(&block->mutex);
/* Remove possible adaptive hash index built on the
page; in the case of AWE the block may not have a
frame at all */
if (block->frame) {
/* The page was declared uninitialized
by buf_LRU_block_remove_hashed_page().
We need to flag the contents of the
page valid (which it still is) in
order to avoid bogus Valgrind
warnings. */
UNIV_MEM_VALID(block->frame, UNIV_PAGE_SIZE);
btr_search_drop_page_hash_index(block->frame);
UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE);
}
ut_a(block->buf_fix_count == 0);
mutex_enter(&(buf_pool->mutex));
mutex_enter(&block->mutex);
buf_LRU_block_free_hashed_page(block);
return(TRUE);
}
/***************************************************************//**
Initializes a buffer to a random combination of hex BA and BE.
Used to initialize allocated memory. */
UNIV_INTERN
void
mem_init_buf(
/*=========*/
byte* buf, /*!< in: pointer to buffer */
ulint n) /*!< in: length of buffer */
{
byte* ptr;
UNIV_MEM_ASSERT_W(buf, n);
for (ptr = buf; ptr < buf + n; ptr++) {
if (ut_rnd_gen_ibool()) {
*ptr = 0xBA;
} else {
*ptr = 0xBE;
}
}
UNIV_MEM_INVALID(buf, n);
}
/***************************************************************//**
Commits a mini-transaction. */
UNIV_INTERN
void
mtr_commit(
/*=======*/
mtr_t* mtr) /*!< in: mini-transaction */
{
ut_ad(mtr);
ut_ad(mtr->magic_n == MTR_MAGIC_N);
ut_ad(mtr->state == MTR_ACTIVE);
ut_ad(!mtr->inside_ibuf);
ut_d(mtr->state = MTR_COMMITTING);
#ifndef UNIV_HOTBACKUP
/* This is a dirty read, for debugging. */
ut_ad(!recv_no_log_write);
if (mtr->modifications && mtr->n_log_recs) {
mtr_log_reserve_and_write(mtr);
}
mtr_memo_pop_all(mtr);
#endif /* !UNIV_HOTBACKUP */
dyn_array_free(&(mtr->memo));
dyn_array_free(&(mtr->log));
#ifdef UNIV_DEBUG_VALGRIND
/* Declare everything uninitialized except
mtr->start_lsn, mtr->end_lsn and mtr->state. */
{
ib_uint64_t start_lsn = mtr->start_lsn;
ib_uint64_t end_lsn = mtr->end_lsn;
UNIV_MEM_INVALID(mtr, sizeof *mtr);
mtr->start_lsn = start_lsn;
mtr->end_lsn = end_lsn;
}
#endif /* UNIV_DEBUG_VALGRIND */
ut_d(mtr->state = MTR_COMMITTED);
}
/**********************************************************************//**
Deallocate a buffer frame of UNIV_PAGE_SIZE. */
static
void
buf_buddy_block_free(
/*=================*/
buf_pool_t* buf_pool, /*!< in: buffer pool instance */
void* buf) /*!< in: buffer frame to deallocate */
{
const ulint fold = BUF_POOL_ZIP_FOLD_PTR(buf);
buf_page_t* bpage;
buf_block_t* block;
ut_ad(buf_pool_mutex_own(buf_pool));
ut_ad(!mutex_own(&buf_pool->zip_mutex));
ut_a(!ut_align_offset(buf, UNIV_PAGE_SIZE));
HASH_SEARCH(hash, buf_pool->zip_hash, fold, buf_page_t*, bpage,
ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_MEMORY
&& bpage->in_zip_hash && !bpage->in_page_hash),
((buf_block_t*) bpage)->frame == buf);
ut_a(bpage);
ut_a(buf_page_get_state(bpage) == BUF_BLOCK_MEMORY);
ut_ad(!bpage->in_page_hash);
ut_ad(bpage->in_zip_hash);
ut_d(bpage->in_zip_hash = FALSE);
HASH_DELETE(buf_page_t, hash, buf_pool->zip_hash, fold, bpage);
ut_d(memset(buf, 0, UNIV_PAGE_SIZE));
UNIV_MEM_INVALID(buf, UNIV_PAGE_SIZE);
block = (buf_block_t*) bpage;
mutex_enter(&block->mutex);
buf_LRU_block_free_non_file_page(block);
mutex_exit(&block->mutex);
ut_ad(buf_pool->buddy_n_frames > 0);
ut_d(buf_pool->buddy_n_frames--);
}
/**********************************************************************//**
Deallocate a block. */
UNIV_INTERN
void
buf_buddy_free_low(
/*===============*/
buf_pool_t* buf_pool, /*!< in: buffer pool instance */
void* buf, /*!< in: block to be freed, must not be
pointed to by the buffer pool */
ulint i) /*!< in: index of buf_pool->zip_free[],
or BUF_BUDDY_SIZES */
{
buf_page_t* bpage;
buf_page_t* buddy;
ut_ad(buf_pool_mutex_own(buf_pool));
ut_ad(!mutex_own(&buf_pool->zip_mutex));
ut_ad(i <= BUF_BUDDY_SIZES);
ut_ad(i >= buf_buddy_get_slot(PAGE_ZIP_MIN_SIZE));
ut_ad(buf_pool->buddy_stat[i].used > 0);
buf_pool->buddy_stat[i].used--;
recombine:
UNIV_MEM_ASSERT_AND_ALLOC(buf, BUF_BUDDY_LOW << i);
((buf_page_t*) buf)->state = BUF_BLOCK_ZIP_FREE;
if (i == BUF_BUDDY_SIZES) {
buf_buddy_block_free(buf_pool, buf);
return;
}
ut_ad(i < BUF_BUDDY_SIZES);
ut_ad(buf == ut_align_down(buf, BUF_BUDDY_LOW << i));
ut_ad(!buf_pool_contains_zip(buf_pool, buf));
/* Do not recombine blocks if there are few free blocks.
We may waste up to 15360*max_len bytes to free blocks
(1024 + 2048 + 4096 + 8192 = 15360) */
if (UT_LIST_GET_LEN(buf_pool->zip_free[i]) < 16) {
goto func_exit;
}
/* Try to combine adjacent blocks. */
buddy = (buf_page_t*) buf_buddy_get(((byte*) buf), BUF_BUDDY_LOW << i);
#ifndef UNIV_DEBUG_VALGRIND
/* When Valgrind instrumentation is not enabled, we can read
buddy->state to quickly determine that a block is not free.
When the block is not free, buddy->state belongs to a compressed
page frame that may be flagged uninitialized in our Valgrind
instrumentation. */
if (buddy->state != BUF_BLOCK_ZIP_FREE) {
goto buddy_nonfree;
}
#endif /* !UNIV_DEBUG_VALGRIND */
for (bpage = UT_LIST_GET_FIRST(buf_pool->zip_free[i]); bpage; ) {
ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_ZIP_FREE);
if (bpage == buddy) {
/* The buddy is free: recombine */
buf_buddy_remove_from_free(buf_pool, bpage, i);
buddy_is_free:
ut_ad(buf_page_get_state(buddy) == BUF_BLOCK_ZIP_FREE);
ut_ad(!buf_pool_contains_zip(buf_pool, buddy));
i++;
buf = ut_align_down(buf, BUF_BUDDY_LOW << i);
goto recombine;
}
ut_a(bpage != buf);
UNIV_MEM_ASSERT_W(bpage, BUF_BUDDY_LOW << i);
bpage = UT_LIST_GET_NEXT(list, bpage);
}
#ifndef UNIV_DEBUG_VALGRIND
buddy_nonfree:
#endif /* !UNIV_DEBUG_VALGRIND */
ut_d(BUF_BUDDY_LIST_VALIDATE(buf_pool, i));
/* The buddy is not free. Is there a free block of this size? */
bpage = UT_LIST_GET_FIRST(buf_pool->zip_free[i]);
if (bpage) {
/* Remove the block from the free list, because a successful
buf_buddy_relocate() will overwrite bpage->list. */
buf_buddy_remove_from_free(buf_pool, bpage, i);
/* Try to relocate the buddy of buf to the free block. */
if (buf_buddy_relocate(buf_pool, buddy, bpage, i)) {
buddy->state = BUF_BLOCK_ZIP_FREE;
goto buddy_is_free;
}
buf_buddy_add_to_free(buf_pool, bpage, i);
}
func_exit:
/* Free the block to the buddy list. */
bpage = buf;
/* Fill large blocks with a constant pattern. */
ut_d(memset(bpage, i, BUF_BUDDY_LOW << i));
UNIV_MEM_INVALID(bpage, BUF_BUDDY_LOW << i);
bpage->state = BUF_BLOCK_ZIP_FREE;
buf_buddy_add_to_free(buf_pool, bpage, i);
}
/**********************************************************************//**
Try to relocate a block.
@return TRUE if relocated */
static
ibool
buf_buddy_relocate(
/*===============*/
buf_pool_t* buf_pool, /*!< in: buffer pool instance */
void* src, /*!< in: block to relocate */
void* dst, /*!< in: free block to relocate to */
ulint i) /*!< in: index of
buf_pool->zip_free[] */
{
buf_page_t* bpage;
const ulint size = BUF_BUDDY_LOW << i;
ullint usec = ut_time_us(NULL);
mutex_t* mutex;
ulint space;
ulint page_no;
ut_ad(buf_pool_mutex_own(buf_pool));
ut_ad(!mutex_own(&buf_pool->zip_mutex));
ut_ad(!ut_align_offset(src, size));
ut_ad(!ut_align_offset(dst, size));
ut_ad(i >= buf_buddy_get_slot(PAGE_ZIP_MIN_SIZE));
UNIV_MEM_ASSERT_W(dst, size);
/* We assume that all memory from buf_buddy_alloc()
is used for compressed page frames. */
/* We look inside the allocated objects returned by
buf_buddy_alloc() and assume that each block is a compressed
page that contains a valid space_id and page_no in the page
header. Should the fields be invalid, we will be unable to
relocate the block. */
/* The src block may be split into smaller blocks,
some of which may be free. Thus, the
mach_read_from_4() calls below may attempt to read
from free memory. The memory is "owned" by the buddy
allocator (and it has been allocated from the buffer
pool), so there is nothing wrong about this. The
mach_read_from_4() calls here will only trigger bogus
Valgrind memcheck warnings in UNIV_DEBUG_VALGRIND builds. */
space = mach_read_from_4((const byte *) src
+ FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
page_no = mach_read_from_4((const byte *) src
+ FIL_PAGE_OFFSET);
/* Suppress Valgrind warnings about conditional jump
on uninitialized value. */
UNIV_MEM_VALID(&space, sizeof space);
UNIV_MEM_VALID(&page_no, sizeof page_no);
bpage = buf_page_hash_get(buf_pool, space, page_no);
if (!bpage || bpage->zip.data != src) {
/* The block has probably been freshly
allocated by buf_LRU_get_free_block() but not
added to buf_pool->page_hash yet. Obviously,
it cannot be relocated. */
return(FALSE);
}
if (page_zip_get_size(&bpage->zip) != size) {
/* The block is of different size. We would
have to relocate all blocks covered by src.
For the sake of simplicity, give up. */
ut_ad(page_zip_get_size(&bpage->zip) < size);
return(FALSE);
}
/* The block must have been allocated, but it may
contain uninitialized data. */
UNIV_MEM_ASSERT_W(src, size);
mutex = buf_page_get_mutex(bpage);
mutex_enter(mutex);
if (buf_page_can_relocate(bpage)) {
/* Relocate the compressed page. */
ut_a(bpage->zip.data == src);
memcpy(dst, src, size);
bpage->zip.data = dst;
mutex_exit(mutex);
UNIV_MEM_INVALID(src, size);
{
buf_buddy_stat_t* buddy_stat
= &buf_pool->buddy_stat[i];
buddy_stat->relocated++;
buddy_stat->relocated_usec
+= ut_time_us(NULL) - usec;
}
return(TRUE);
}
mutex_exit(mutex);
return(FALSE);
}
/******************************************************************//**
Creates, or rather, initializes an rw-lock object in a specified memory
location (which must be appropriately aligned). The rw-lock is initialized
to the non-locked state. Explicit freeing of the rw-lock with rw_lock_free
is necessary only if the memory block containing it is freed. */
UNIV_INTERN
void
rw_lock_create_func(
/*================*/
rw_lock_t* lock, /*!< in: pointer to memory */
#ifdef UNIV_DEBUG
# ifdef UNIV_SYNC_DEBUG
ulint level, /*!< in: level */
# endif /* UNIV_SYNC_DEBUG */
const char* cmutex_name, /*!< in: mutex name */
#endif /* UNIV_DEBUG */
const char* cfile_name, /*!< in: file name where created */
ulint cline) /*!< in: file line where created */
{
/* If this is the very first time a synchronization object is
created, then the following call initializes the sync system. */
#ifndef INNODB_RW_LOCKS_USE_ATOMICS
mutex_create(rw_lock_mutex_key, rw_lock_get_mutex(lock),
SYNC_NO_ORDER_CHECK);
lock->mutex.cfile_name = cfile_name;
lock->mutex.cline = cline;
ut_d(lock->mutex.cmutex_name = cmutex_name);
ut_d(lock->mutex.mutex_type = 1);
#else /* INNODB_RW_LOCKS_USE_ATOMICS */
# ifdef UNIV_DEBUG
UT_NOT_USED(cmutex_name);
# endif
#endif /* INNODB_RW_LOCKS_USE_ATOMICS */
lock->lock_word = X_LOCK_DECR;
lock->waiters = 0;
/* We set this value to signify that lock->writer_thread
contains garbage at initialization and cannot be used for
recursive x-locking. */
lock->recursive = FALSE;
/* Silence Valgrind when UNIV_DEBUG_VALGRIND is not enabled. */
memset((void*) &lock->writer_thread, 0, sizeof lock->writer_thread);
UNIV_MEM_INVALID(&lock->writer_thread, sizeof lock->writer_thread);
#ifdef UNIV_SYNC_DEBUG
UT_LIST_INIT(lock->debug_list);
lock->level = level;
#endif /* UNIV_SYNC_DEBUG */
ut_d(lock->magic_n = RW_LOCK_MAGIC_N);
lock->cfile_name = cfile_name;
lock->cline = (unsigned int) cline;
lock->count_os_wait = 0;
lock->last_s_file_name = "not yet reserved";
lock->last_x_file_name = "not yet reserved";
lock->last_s_line = 0;
lock->last_x_line = 0;
lock->event = os_event_create(NULL);
lock->wait_ex_event = os_event_create(NULL);
mutex_enter(&rw_lock_list_mutex);
ut_ad(UT_LIST_GET_FIRST(rw_lock_list) == NULL
|| UT_LIST_GET_FIRST(rw_lock_list)->magic_n == RW_LOCK_MAGIC_N);
UT_LIST_ADD_FIRST(list, rw_lock_list, lock);
mutex_exit(&rw_lock_list_mutex);
}
/***************************************************************//**
Creates a memory heap block where data can be allocated.
@return own: memory heap block, NULL if did not succeed (only possible
for MEM_HEAP_BTR_SEARCH type heaps) */
UNIV_INTERN
mem_block_t*
mem_heap_create_block(
/*==================*/
mem_heap_t* heap, /*!< in: memory heap or NULL if first block
should be created */
ulint n, /*!< in: number of bytes needed for user data */
ulint type, /*!< in: type of heap: MEM_HEAP_DYNAMIC or
MEM_HEAP_BUFFER */
const char* file_name,/*!< in: file name where created */
ulint line) /*!< in: line where created */
{
#ifndef UNIV_HOTBACKUP
buf_block_t* buf_block = NULL;
#endif /* !UNIV_HOTBACKUP */
mem_block_t* block;
ulint len;
ut_ad((type == MEM_HEAP_DYNAMIC) || (type == MEM_HEAP_BUFFER)
|| (type == MEM_HEAP_BUFFER + MEM_HEAP_BTR_SEARCH));
if (heap && heap->magic_n != MEM_BLOCK_MAGIC_N) {
mem_analyze_corruption(heap);
}
/* In dynamic allocation, calculate the size: block header + data. */
len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n);
#ifndef UNIV_HOTBACKUP
if (type == MEM_HEAP_DYNAMIC || len < UNIV_PAGE_SIZE / 2) {
ut_ad(type == MEM_HEAP_DYNAMIC || n <= MEM_MAX_ALLOC_IN_BUF);
block = mem_area_alloc(&len, mem_comm_pool);
} else {
len = UNIV_PAGE_SIZE;
if ((type & MEM_HEAP_BTR_SEARCH) && heap) {
/* We cannot allocate the block from the
buffer pool, but must get the free block from
the heap header free block field */
buf_block = heap->free_block;
heap->free_block = NULL;
if (UNIV_UNLIKELY(!buf_block)) {
return(NULL);
}
} else {
buf_block = buf_block_alloc(NULL, 0);
}
block = (mem_block_t*) buf_block->frame;
}
ut_ad(block);
block->buf_block = buf_block;
block->free_block = NULL;
#else /* !UNIV_HOTBACKUP */
len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n);
block = ut_malloc(len);
ut_ad(block);
#endif /* !UNIV_HOTBACKUP */
block->magic_n = MEM_BLOCK_MAGIC_N;
ut_strlcpy_rev(block->file_name, file_name, sizeof(block->file_name));
block->line = line;
#ifdef MEM_PERIODIC_CHECK
mem_pool_mutex_enter();
if (!mem_block_list_inited) {
mem_block_list_inited = TRUE;
UT_LIST_INIT(mem_block_list);
}
UT_LIST_ADD_LAST(mem_block_list, mem_block_list, block);
mem_pool_mutex_exit();
#endif
mem_block_set_len(block, len);
mem_block_set_type(block, type);
mem_block_set_free(block, MEM_BLOCK_HEADER_SIZE);
mem_block_set_start(block, MEM_BLOCK_HEADER_SIZE);
if (UNIV_UNLIKELY(heap == NULL)) {
/* This is the first block of the heap. The field
total_size should be initialized here */
block->total_size = len;
} else {
/* Not the first allocation for the heap. This block's
total_length field should be set to undefined. */
ut_d(block->total_size = ULINT_UNDEFINED);
UNIV_MEM_INVALID(&block->total_size,
sizeof block->total_size);
heap->total_size += len;
}
ut_ad((ulint)MEM_BLOCK_HEADER_SIZE < len);
return(block);
}