/********************************************************************** Takes a block out of the LRU list and page hash table and sets the block state to BUF_BLOCK_REMOVE_HASH. */ static void buf_LRU_block_remove_hashed_page( /*=============================*/ buf_block_t* block) /* in: block, must contain a file page and be in a state where it can be freed; there may or may not be a hash index to the page */ { ut_ad(mutex_own(&(buf_pool->mutex))); ut_ad(mutex_own(&block->mutex)); ut_ad(block); ut_a(block->state == BUF_BLOCK_FILE_PAGE); ut_a(block->io_fix == 0); ut_a(block->buf_fix_count == 0); ut_a(ut_dulint_cmp(block->oldest_modification, ut_dulint_zero) == 0); buf_LRU_remove_block(block); buf_pool->freed_page_clock += 1; /* Note that if AWE is enabled the block may not have a frame at all */ buf_block_modify_clock_inc(block); if (block != buf_page_hash_get(block->space, block->offset)) { fprintf(stderr, "InnoDB: Error: page %lu %lu not found" " in the hash table\n", (ulong) block->space, (ulong) block->offset); if (buf_page_hash_get(block->space, block->offset)) { fprintf(stderr, "InnoDB: In hash table we find block" " %p of %lu %lu which is not %p\n", (void*) buf_page_hash_get (block->space, block->offset), (ulong) buf_page_hash_get (block->space, block->offset)->space, (ulong) buf_page_hash_get (block->space, block->offset)->offset, (void*) block); } #ifdef UNIV_DEBUG buf_print(); buf_LRU_print(); buf_validate(); buf_LRU_validate(); #endif ut_a(0); } HASH_DELETE(buf_block_t, hash, buf_pool->page_hash, buf_page_address_fold(block->space, block->offset), block); UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE); block->state = BUF_BLOCK_REMOVE_HASH; }
ibool buf_LRU_free_block( /*===============*/ /* out: TRUE if freed */ buf_block_t* block) /* in/out: block to be freed */ { if (!buf_flush_ready_for_replace(block)) { return(FALSE); } #ifdef UNIV_DEBUG if (buf_debug_prints) { fprintf(stderr, "Putting space %lu page %lu" " to free list\n", (ulong) block->space, (ulong) block->offset); } #endif /* UNIV_DEBUG */ buf_LRU_block_remove_hashed_page(block); mutex_exit(&(buf_pool->mutex)); mutex_exit(&block->mutex); /* Remove possible adaptive hash index built on the page; in the case of AWE the block may not have a frame at all */ if (block->frame) { /* The page was declared uninitialized by buf_LRU_block_remove_hashed_page(). We need to flag the contents of the page valid (which it still is) in order to avoid bogus Valgrind warnings. */ UNIV_MEM_VALID(block->frame, UNIV_PAGE_SIZE); btr_search_drop_page_hash_index(block->frame); UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE); } ut_a(block->buf_fix_count == 0); mutex_enter(&(buf_pool->mutex)); mutex_enter(&block->mutex); buf_LRU_block_free_hashed_page(block); return(TRUE); }
/***************************************************************//** Initializes a buffer to a random combination of hex BA and BE. Used to initialize allocated memory. */ UNIV_INTERN void mem_init_buf( /*=========*/ byte* buf, /*!< in: pointer to buffer */ ulint n) /*!< in: length of buffer */ { byte* ptr; UNIV_MEM_ASSERT_W(buf, n); for (ptr = buf; ptr < buf + n; ptr++) { if (ut_rnd_gen_ibool()) { *ptr = 0xBA; } else { *ptr = 0xBE; } } UNIV_MEM_INVALID(buf, n); }
/***************************************************************//** Commits a mini-transaction. */ UNIV_INTERN void mtr_commit( /*=======*/ mtr_t* mtr) /*!< in: mini-transaction */ { ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); ut_ad(!mtr->inside_ibuf); ut_d(mtr->state = MTR_COMMITTING); #ifndef UNIV_HOTBACKUP /* This is a dirty read, for debugging. */ ut_ad(!recv_no_log_write); if (mtr->modifications && mtr->n_log_recs) { mtr_log_reserve_and_write(mtr); } mtr_memo_pop_all(mtr); #endif /* !UNIV_HOTBACKUP */ dyn_array_free(&(mtr->memo)); dyn_array_free(&(mtr->log)); #ifdef UNIV_DEBUG_VALGRIND /* Declare everything uninitialized except mtr->start_lsn, mtr->end_lsn and mtr->state. */ { ib_uint64_t start_lsn = mtr->start_lsn; ib_uint64_t end_lsn = mtr->end_lsn; UNIV_MEM_INVALID(mtr, sizeof *mtr); mtr->start_lsn = start_lsn; mtr->end_lsn = end_lsn; } #endif /* UNIV_DEBUG_VALGRIND */ ut_d(mtr->state = MTR_COMMITTED); }
/**********************************************************************//** Deallocate a buffer frame of UNIV_PAGE_SIZE. */ static void buf_buddy_block_free( /*=================*/ buf_pool_t* buf_pool, /*!< in: buffer pool instance */ void* buf) /*!< in: buffer frame to deallocate */ { const ulint fold = BUF_POOL_ZIP_FOLD_PTR(buf); buf_page_t* bpage; buf_block_t* block; ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); ut_a(!ut_align_offset(buf, UNIV_PAGE_SIZE)); HASH_SEARCH(hash, buf_pool->zip_hash, fold, buf_page_t*, bpage, ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_MEMORY && bpage->in_zip_hash && !bpage->in_page_hash), ((buf_block_t*) bpage)->frame == buf); ut_a(bpage); ut_a(buf_page_get_state(bpage) == BUF_BLOCK_MEMORY); ut_ad(!bpage->in_page_hash); ut_ad(bpage->in_zip_hash); ut_d(bpage->in_zip_hash = FALSE); HASH_DELETE(buf_page_t, hash, buf_pool->zip_hash, fold, bpage); ut_d(memset(buf, 0, UNIV_PAGE_SIZE)); UNIV_MEM_INVALID(buf, UNIV_PAGE_SIZE); block = (buf_block_t*) bpage; mutex_enter(&block->mutex); buf_LRU_block_free_non_file_page(block); mutex_exit(&block->mutex); ut_ad(buf_pool->buddy_n_frames > 0); ut_d(buf_pool->buddy_n_frames--); }
/**********************************************************************//** Deallocate a block. */ UNIV_INTERN void buf_buddy_free_low( /*===============*/ buf_pool_t* buf_pool, /*!< in: buffer pool instance */ void* buf, /*!< in: block to be freed, must not be pointed to by the buffer pool */ ulint i) /*!< in: index of buf_pool->zip_free[], or BUF_BUDDY_SIZES */ { buf_page_t* bpage; buf_page_t* buddy; ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); ut_ad(i <= BUF_BUDDY_SIZES); ut_ad(i >= buf_buddy_get_slot(PAGE_ZIP_MIN_SIZE)); ut_ad(buf_pool->buddy_stat[i].used > 0); buf_pool->buddy_stat[i].used--; recombine: UNIV_MEM_ASSERT_AND_ALLOC(buf, BUF_BUDDY_LOW << i); ((buf_page_t*) buf)->state = BUF_BLOCK_ZIP_FREE; if (i == BUF_BUDDY_SIZES) { buf_buddy_block_free(buf_pool, buf); return; } ut_ad(i < BUF_BUDDY_SIZES); ut_ad(buf == ut_align_down(buf, BUF_BUDDY_LOW << i)); ut_ad(!buf_pool_contains_zip(buf_pool, buf)); /* Do not recombine blocks if there are few free blocks. We may waste up to 15360*max_len bytes to free blocks (1024 + 2048 + 4096 + 8192 = 15360) */ if (UT_LIST_GET_LEN(buf_pool->zip_free[i]) < 16) { goto func_exit; } /* Try to combine adjacent blocks. */ buddy = (buf_page_t*) buf_buddy_get(((byte*) buf), BUF_BUDDY_LOW << i); #ifndef UNIV_DEBUG_VALGRIND /* When Valgrind instrumentation is not enabled, we can read buddy->state to quickly determine that a block is not free. When the block is not free, buddy->state belongs to a compressed page frame that may be flagged uninitialized in our Valgrind instrumentation. */ if (buddy->state != BUF_BLOCK_ZIP_FREE) { goto buddy_nonfree; } #endif /* !UNIV_DEBUG_VALGRIND */ for (bpage = UT_LIST_GET_FIRST(buf_pool->zip_free[i]); bpage; ) { ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_ZIP_FREE); if (bpage == buddy) { /* The buddy is free: recombine */ buf_buddy_remove_from_free(buf_pool, bpage, i); buddy_is_free: ut_ad(buf_page_get_state(buddy) == BUF_BLOCK_ZIP_FREE); ut_ad(!buf_pool_contains_zip(buf_pool, buddy)); i++; buf = ut_align_down(buf, BUF_BUDDY_LOW << i); goto recombine; } ut_a(bpage != buf); UNIV_MEM_ASSERT_W(bpage, BUF_BUDDY_LOW << i); bpage = UT_LIST_GET_NEXT(list, bpage); } #ifndef UNIV_DEBUG_VALGRIND buddy_nonfree: #endif /* !UNIV_DEBUG_VALGRIND */ ut_d(BUF_BUDDY_LIST_VALIDATE(buf_pool, i)); /* The buddy is not free. Is there a free block of this size? */ bpage = UT_LIST_GET_FIRST(buf_pool->zip_free[i]); if (bpage) { /* Remove the block from the free list, because a successful buf_buddy_relocate() will overwrite bpage->list. */ buf_buddy_remove_from_free(buf_pool, bpage, i); /* Try to relocate the buddy of buf to the free block. */ if (buf_buddy_relocate(buf_pool, buddy, bpage, i)) { buddy->state = BUF_BLOCK_ZIP_FREE; goto buddy_is_free; } buf_buddy_add_to_free(buf_pool, bpage, i); } func_exit: /* Free the block to the buddy list. */ bpage = buf; /* Fill large blocks with a constant pattern. */ ut_d(memset(bpage, i, BUF_BUDDY_LOW << i)); UNIV_MEM_INVALID(bpage, BUF_BUDDY_LOW << i); bpage->state = BUF_BLOCK_ZIP_FREE; buf_buddy_add_to_free(buf_pool, bpage, i); }
/**********************************************************************//** Try to relocate a block. @return TRUE if relocated */ static ibool buf_buddy_relocate( /*===============*/ buf_pool_t* buf_pool, /*!< in: buffer pool instance */ void* src, /*!< in: block to relocate */ void* dst, /*!< in: free block to relocate to */ ulint i) /*!< in: index of buf_pool->zip_free[] */ { buf_page_t* bpage; const ulint size = BUF_BUDDY_LOW << i; ullint usec = ut_time_us(NULL); mutex_t* mutex; ulint space; ulint page_no; ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); ut_ad(!ut_align_offset(src, size)); ut_ad(!ut_align_offset(dst, size)); ut_ad(i >= buf_buddy_get_slot(PAGE_ZIP_MIN_SIZE)); UNIV_MEM_ASSERT_W(dst, size); /* We assume that all memory from buf_buddy_alloc() is used for compressed page frames. */ /* We look inside the allocated objects returned by buf_buddy_alloc() and assume that each block is a compressed page that contains a valid space_id and page_no in the page header. Should the fields be invalid, we will be unable to relocate the block. */ /* The src block may be split into smaller blocks, some of which may be free. Thus, the mach_read_from_4() calls below may attempt to read from free memory. The memory is "owned" by the buddy allocator (and it has been allocated from the buffer pool), so there is nothing wrong about this. The mach_read_from_4() calls here will only trigger bogus Valgrind memcheck warnings in UNIV_DEBUG_VALGRIND builds. */ space = mach_read_from_4((const byte *) src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); page_no = mach_read_from_4((const byte *) src + FIL_PAGE_OFFSET); /* Suppress Valgrind warnings about conditional jump on uninitialized value. */ UNIV_MEM_VALID(&space, sizeof space); UNIV_MEM_VALID(&page_no, sizeof page_no); bpage = buf_page_hash_get(buf_pool, space, page_no); if (!bpage || bpage->zip.data != src) { /* The block has probably been freshly allocated by buf_LRU_get_free_block() but not added to buf_pool->page_hash yet. Obviously, it cannot be relocated. */ return(FALSE); } if (page_zip_get_size(&bpage->zip) != size) { /* The block is of different size. We would have to relocate all blocks covered by src. For the sake of simplicity, give up. */ ut_ad(page_zip_get_size(&bpage->zip) < size); return(FALSE); } /* The block must have been allocated, but it may contain uninitialized data. */ UNIV_MEM_ASSERT_W(src, size); mutex = buf_page_get_mutex(bpage); mutex_enter(mutex); if (buf_page_can_relocate(bpage)) { /* Relocate the compressed page. */ ut_a(bpage->zip.data == src); memcpy(dst, src, size); bpage->zip.data = dst; mutex_exit(mutex); UNIV_MEM_INVALID(src, size); { buf_buddy_stat_t* buddy_stat = &buf_pool->buddy_stat[i]; buddy_stat->relocated++; buddy_stat->relocated_usec += ut_time_us(NULL) - usec; } return(TRUE); } mutex_exit(mutex); return(FALSE); }
/******************************************************************//** Creates, or rather, initializes an rw-lock object in a specified memory location (which must be appropriately aligned). The rw-lock is initialized to the non-locked state. Explicit freeing of the rw-lock with rw_lock_free is necessary only if the memory block containing it is freed. */ UNIV_INTERN void rw_lock_create_func( /*================*/ rw_lock_t* lock, /*!< in: pointer to memory */ #ifdef UNIV_DEBUG # ifdef UNIV_SYNC_DEBUG ulint level, /*!< in: level */ # endif /* UNIV_SYNC_DEBUG */ const char* cmutex_name, /*!< in: mutex name */ #endif /* UNIV_DEBUG */ const char* cfile_name, /*!< in: file name where created */ ulint cline) /*!< in: file line where created */ { /* If this is the very first time a synchronization object is created, then the following call initializes the sync system. */ #ifndef INNODB_RW_LOCKS_USE_ATOMICS mutex_create(rw_lock_mutex_key, rw_lock_get_mutex(lock), SYNC_NO_ORDER_CHECK); lock->mutex.cfile_name = cfile_name; lock->mutex.cline = cline; ut_d(lock->mutex.cmutex_name = cmutex_name); ut_d(lock->mutex.mutex_type = 1); #else /* INNODB_RW_LOCKS_USE_ATOMICS */ # ifdef UNIV_DEBUG UT_NOT_USED(cmutex_name); # endif #endif /* INNODB_RW_LOCKS_USE_ATOMICS */ lock->lock_word = X_LOCK_DECR; lock->waiters = 0; /* We set this value to signify that lock->writer_thread contains garbage at initialization and cannot be used for recursive x-locking. */ lock->recursive = FALSE; /* Silence Valgrind when UNIV_DEBUG_VALGRIND is not enabled. */ memset((void*) &lock->writer_thread, 0, sizeof lock->writer_thread); UNIV_MEM_INVALID(&lock->writer_thread, sizeof lock->writer_thread); #ifdef UNIV_SYNC_DEBUG UT_LIST_INIT(lock->debug_list); lock->level = level; #endif /* UNIV_SYNC_DEBUG */ ut_d(lock->magic_n = RW_LOCK_MAGIC_N); lock->cfile_name = cfile_name; lock->cline = (unsigned int) cline; lock->count_os_wait = 0; lock->last_s_file_name = "not yet reserved"; lock->last_x_file_name = "not yet reserved"; lock->last_s_line = 0; lock->last_x_line = 0; lock->event = os_event_create(NULL); lock->wait_ex_event = os_event_create(NULL); mutex_enter(&rw_lock_list_mutex); ut_ad(UT_LIST_GET_FIRST(rw_lock_list) == NULL || UT_LIST_GET_FIRST(rw_lock_list)->magic_n == RW_LOCK_MAGIC_N); UT_LIST_ADD_FIRST(list, rw_lock_list, lock); mutex_exit(&rw_lock_list_mutex); }
/***************************************************************//** Creates a memory heap block where data can be allocated. @return own: memory heap block, NULL if did not succeed (only possible for MEM_HEAP_BTR_SEARCH type heaps) */ UNIV_INTERN mem_block_t* mem_heap_create_block( /*==================*/ mem_heap_t* heap, /*!< in: memory heap or NULL if first block should be created */ ulint n, /*!< in: number of bytes needed for user data */ ulint type, /*!< in: type of heap: MEM_HEAP_DYNAMIC or MEM_HEAP_BUFFER */ const char* file_name,/*!< in: file name where created */ ulint line) /*!< in: line where created */ { #ifndef UNIV_HOTBACKUP buf_block_t* buf_block = NULL; #endif /* !UNIV_HOTBACKUP */ mem_block_t* block; ulint len; ut_ad((type == MEM_HEAP_DYNAMIC) || (type == MEM_HEAP_BUFFER) || (type == MEM_HEAP_BUFFER + MEM_HEAP_BTR_SEARCH)); if (heap && heap->magic_n != MEM_BLOCK_MAGIC_N) { mem_analyze_corruption(heap); } /* In dynamic allocation, calculate the size: block header + data. */ len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n); #ifndef UNIV_HOTBACKUP if (type == MEM_HEAP_DYNAMIC || len < UNIV_PAGE_SIZE / 2) { ut_ad(type == MEM_HEAP_DYNAMIC || n <= MEM_MAX_ALLOC_IN_BUF); block = mem_area_alloc(&len, mem_comm_pool); } else { len = UNIV_PAGE_SIZE; if ((type & MEM_HEAP_BTR_SEARCH) && heap) { /* We cannot allocate the block from the buffer pool, but must get the free block from the heap header free block field */ buf_block = heap->free_block; heap->free_block = NULL; if (UNIV_UNLIKELY(!buf_block)) { return(NULL); } } else { buf_block = buf_block_alloc(NULL, 0); } block = (mem_block_t*) buf_block->frame; } ut_ad(block); block->buf_block = buf_block; block->free_block = NULL; #else /* !UNIV_HOTBACKUP */ len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n); block = ut_malloc(len); ut_ad(block); #endif /* !UNIV_HOTBACKUP */ block->magic_n = MEM_BLOCK_MAGIC_N; ut_strlcpy_rev(block->file_name, file_name, sizeof(block->file_name)); block->line = line; #ifdef MEM_PERIODIC_CHECK mem_pool_mutex_enter(); if (!mem_block_list_inited) { mem_block_list_inited = TRUE; UT_LIST_INIT(mem_block_list); } UT_LIST_ADD_LAST(mem_block_list, mem_block_list, block); mem_pool_mutex_exit(); #endif mem_block_set_len(block, len); mem_block_set_type(block, type); mem_block_set_free(block, MEM_BLOCK_HEADER_SIZE); mem_block_set_start(block, MEM_BLOCK_HEADER_SIZE); if (UNIV_UNLIKELY(heap == NULL)) { /* This is the first block of the heap. The field total_size should be initialized here */ block->total_size = len; } else { /* Not the first allocation for the heap. This block's total_length field should be set to undefined. */ ut_d(block->total_size = ULINT_UNDEFINED); UNIV_MEM_INVALID(&block->total_size, sizeof block->total_size); heap->total_size += len; } ut_ad((ulint)MEM_BLOCK_HEADER_SIZE < len); return(block); }