bool _Heap_Size_of_alloc_area(
Heap_Control *heap,
void *alloc_begin_ptr,
uintptr_t *alloc_size
)
{
uintptr_t const page_size = heap->page_size;
uintptr_t const alloc_begin = (uintptr_t) alloc_begin_ptr;
Heap_Block *block = _Heap_Block_of_alloc_area( alloc_begin, page_size );
Heap_Block *next_block = NULL;
uintptr_t block_size = 0;
if ( !_Heap_Is_block_in_heap( heap, block ) ) {
return false;
}
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
if (
!_Heap_Is_block_in_heap( heap, next_block )
|| !_Heap_Is_prev_used( next_block )
) {
return false;
}
*alloc_size = (uintptr_t) next_block + HEAP_BLOCK_SIZE_OFFSET - alloc_begin;
return true;
}
Heap_Resize_status _Heap_Resize_block(
Heap_Control *heap,
void *alloc_begin_ptr,
uintptr_t new_alloc_size,
uintptr_t *old_size,
uintptr_t *new_size
)
{
uintptr_t const page_size = heap->page_size;
uintptr_t const alloc_begin = (uintptr_t) alloc_begin_ptr;
Heap_Block *const block = _Heap_Block_of_alloc_area( alloc_begin, page_size );
*old_size = 0;
*new_size = 0;
if ( _Heap_Is_block_in_heap( heap, block ) ) {
_Heap_Protection_block_check( heap, block );
return _Heap_Resize_block_checked(
heap,
block,
alloc_begin,
new_alloc_size,
old_size,
new_size
);
}
return HEAP_RESIZE_FATAL_ERROR;
}
static Heap_Resize_status _Heap_Resize_block_checked(
Heap_Control *heap,
Heap_Block *block,
uintptr_t alloc_begin,
uintptr_t new_alloc_size,
uintptr_t *old_size,
uintptr_t *new_size
)
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t const block_begin = (uintptr_t) block;
uintptr_t block_size = _Heap_Block_size( block );
uintptr_t block_end = block_begin + block_size;
uintptr_t alloc_size = block_end - alloc_begin + HEAP_ALLOC_BONUS;
Heap_Block *next_block = _Heap_Block_at( block, block_size );
uintptr_t next_block_size = _Heap_Block_size( next_block );
bool next_block_is_free = _Heap_Is_free( next_block );
_HAssert( _Heap_Is_block_in_heap( heap, next_block ) );
_HAssert( _Heap_Is_prev_used( next_block ) );
*old_size = alloc_size;
if ( next_block_is_free ) {
block_size += next_block_size;
alloc_size += next_block_size;
}
if ( new_alloc_size > alloc_size ) {
return HEAP_RESIZE_UNSATISFIED;
}
if ( next_block_is_free ) {
_Heap_Block_set_size( block, block_size );
_Heap_Free_list_remove( next_block );
next_block = _Heap_Block_at( block, block_size );
next_block->size_and_flag |= HEAP_PREV_BLOCK_USED;
/* Statistics */
--stats->free_blocks;
stats->free_size -= next_block_size;
}
block = _Heap_Block_allocate( heap, block, alloc_begin, new_alloc_size );
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
*new_size = (uintptr_t) next_block - alloc_begin + HEAP_ALLOC_BONUS;
/* Statistics */
++stats->resizes;
return HEAP_RESIZE_SUCCESSFUL;
}
static bool _Heap_Protection_free_delayed_blocks(
Heap_Control *heap,
uintptr_t alloc_begin
)
{
bool search_again = false;
uintptr_t const blocks_to_free_count =
(heap->Protection.delayed_free_block_count
+ heap->Protection.delayed_free_fraction - 1)
/ heap->Protection.delayed_free_fraction;
if ( alloc_begin == 0 && blocks_to_free_count > 0 ) {
Heap_Block *block_to_free = heap->Protection.first_delayed_free_block;
uintptr_t count = 0;
for ( count = 0; count < blocks_to_free_count; ++count ) {
Heap_Block *next_block_to_free;
if ( !_Heap_Is_block_in_heap( heap, block_to_free ) ) {
_Heap_Protection_block_error( heap, block_to_free );
}
next_block_to_free =
block_to_free->Protection_begin.next_delayed_free_block;
block_to_free->Protection_begin.next_delayed_free_block =
HEAP_PROTECTION_OBOLUS;
_Heap_Free(
heap,
(void *) _Heap_Alloc_area_of_block( block_to_free )
);
block_to_free = next_block_to_free;
}
heap->Protection.delayed_free_block_count -= blocks_to_free_count;
heap->Protection.first_delayed_free_block = block_to_free;
search_again = true;
}
return search_again;
}
bool _Heap_Free( Heap_Control *heap, void *alloc_begin_ptr )
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t alloc_begin = (uintptr_t) alloc_begin_ptr;
Heap_Block *block =
_Heap_Block_of_alloc_area( alloc_begin, heap->page_size );
Heap_Block *next_block = NULL;
uintptr_t block_size = 0;
uintptr_t next_block_size = 0;
bool next_is_free = false;
if ( !_Heap_Is_block_in_heap( heap, block ) ) {
return false;
}
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
if ( !_Heap_Is_block_in_heap( heap, next_block ) ) {
_HAssert( false );
return false;
}
if ( !_Heap_Is_prev_used( next_block ) ) {
_HAssert( false );
return false;
}
next_block_size = _Heap_Block_size( next_block );
next_is_free = next_block != heap->last_block
&& !_Heap_Is_prev_used( _Heap_Block_at( next_block, next_block_size ));
if ( !_Heap_Is_prev_used( block ) ) {
uintptr_t const prev_size = block->prev_size;
Heap_Block * const prev_block = _Heap_Block_at( block, -prev_size );
if ( !_Heap_Is_block_in_heap( heap, prev_block ) ) {
_HAssert( false );
return( false );
}
/* As we always coalesce free blocks, the block that preceedes prev_block
must have been used. */
if ( !_Heap_Is_prev_used ( prev_block) ) {
_HAssert( false );
return( false );
}
if ( next_is_free ) { /* coalesce both */
uintptr_t const size = block_size + prev_size + next_block_size;
_Heap_Free_list_remove( next_block );
stats->free_blocks -= 1;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( prev_block, size );
_HAssert(!_Heap_Is_prev_used( next_block));
next_block->prev_size = size;
} else { /* coalesce prev */
uintptr_t const size = block_size + prev_size;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = size;
}
} else if ( next_is_free ) { /* coalesce next */
uintptr_t const size = block_size + next_block_size;
_Heap_Free_list_replace( next_block, block );
block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( block, size );
next_block->prev_size = size;
} else { /* no coalesce */
/* Add 'block' to the head of the free blocks list as it tends to
produce less fragmentation than adding to the tail. */
_Heap_Free_list_insert_after( _Heap_Free_list_head( heap), block );
block->size_and_flag = block_size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = block_size;
/* Statistics */
++stats->free_blocks;
if ( stats->max_free_blocks < stats->free_blocks ) {
stats->max_free_blocks = stats->free_blocks;
}
}
/* Statistics */
--stats->used_blocks;
++stats->frees;
stats->free_size += block_size;
return( true );
}
static bool _Heap_Walk_check_free_list(
int source,
Heap_Walk_printer printer,
Heap_Control *heap
)
{
uintptr_t const page_size = heap->page_size;
const Heap_Block *const free_list_tail = _Heap_Free_list_tail( heap );
const Heap_Block *const first_free_block = _Heap_Free_list_first( heap );
const Heap_Block *prev_block = free_list_tail;
const Heap_Block *free_block = first_free_block;
while ( free_block != free_list_tail ) {
if ( !_Heap_Is_block_in_heap( heap, free_block ) ) {
(*printer)(
source,
true,
"free block 0x%08x: not in heap\n",
free_block
);
return false;
}
if (
!_Heap_Is_aligned( _Heap_Alloc_area_of_block( free_block ), page_size )
) {
(*printer)(
source,
true,
"free block 0x%08x: alloc area not page aligned\n",
free_block
);
return false;
}
if ( _Heap_Is_used( free_block ) ) {
(*printer)(
source,
true,
"free block 0x%08x: is used\n",
free_block
);
return false;
}
if ( free_block->prev != prev_block ) {
(*printer)(
source,
true,
"free block 0x%08x: invalid previous block 0x%08x\n",
free_block,
free_block->prev
);
return false;
}
prev_block = free_block;
free_block = free_block->next;
}
return true;
}
bool _Heap_Walk(
Heap_Control *heap,
int source,
bool dump
)
{
uintptr_t const page_size = heap->page_size;
uintptr_t const min_block_size = heap->min_block_size;
Heap_Block *const first_block = heap->first_block;
Heap_Block *const last_block = heap->last_block;
Heap_Block *block = first_block;
Heap_Walk_printer printer = dump ?
_Heap_Walk_print : _Heap_Walk_print_nothing;
if ( !_System_state_Is_up( _System_state_Get() ) ) {
return true;
}
if ( !_Heap_Walk_check_control( source, printer, heap ) ) {
return false;
}
do {
uintptr_t const block_begin = (uintptr_t) block;
uintptr_t const block_size = _Heap_Block_size( block );
bool const prev_used = _Heap_Is_prev_used( block );
Heap_Block *const next_block = _Heap_Block_at( block, block_size );
uintptr_t const next_block_begin = (uintptr_t) next_block;
bool const is_not_last_block = block != last_block;
if ( !_Heap_Is_block_in_heap( heap, next_block ) ) {
(*printer)(
source,
true,
"block 0x%08x: next block 0x%08x not in heap\n",
block,
next_block
);
return false;
}
if ( !_Heap_Is_aligned( block_size, page_size ) && is_not_last_block ) {
(*printer)(
source,
true,
"block 0x%08x: block size %u not page aligned\n",
block,
block_size
);
return false;
}
if ( block_size < min_block_size && is_not_last_block ) {
(*printer)(
source,
true,
"block 0x%08x: size %u < min block size %u\n",
block,
block_size,
min_block_size
);
return false;
}
if ( next_block_begin <= block_begin && is_not_last_block ) {
(*printer)(
source,
true,
"block 0x%08x: next block 0x%08x is not a successor\n",
block,
next_block
);
return false;
}
if ( !_Heap_Is_prev_used( next_block ) ) {
if ( !_Heap_Walk_check_free_block( source, printer, heap, block ) ) {
return false;
}
} else if (prev_used) {
(*printer)(
source,
false,
"block 0x%08x: size %u\n",
block,
block_size
);
} else {
(*printer)(
source,
false,
"block 0x%08x: size %u, prev_size %u\n",
block,
block_size,
block->prev_size
);
}
block = next_block;
} while ( block != first_block );
return true;
}
bool _Heap_Free( Heap_Control *heap, void *alloc_begin_ptr )
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t alloc_begin;
Heap_Block *block;
Heap_Block *next_block = NULL;
uintptr_t block_size = 0;
uintptr_t next_block_size = 0;
bool next_is_free = false;
/*
* If NULL return true so a free on NULL is considered a valid release. This
* is a special case that could be handled by the in heap check how-ever that
* would result in false being returned which is wrong.
*/
if ( alloc_begin_ptr == NULL ) {
return true;
}
alloc_begin = (uintptr_t) alloc_begin_ptr;
block = _Heap_Block_of_alloc_area( alloc_begin, heap->page_size );
if ( !_Heap_Is_block_in_heap( heap, block ) ) {
return false;
}
_Heap_Protection_block_check( heap, block );
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
if ( !_Heap_Is_block_in_heap( heap, next_block ) ) {
return false;
}
_Heap_Protection_block_check( heap, next_block );
if ( !_Heap_Is_prev_used( next_block ) ) {
_Heap_Protection_block_error( heap, block );
return false;
}
if ( !_Heap_Protection_determine_block_free( heap, block ) ) {
return true;
}
next_block_size = _Heap_Block_size( next_block );
next_is_free = next_block != heap->last_block
&& !_Heap_Is_prev_used( _Heap_Block_at( next_block, next_block_size ));
if ( !_Heap_Is_prev_used( block ) ) {
uintptr_t const prev_size = block->prev_size;
Heap_Block * const prev_block = _Heap_Block_at( block, -prev_size );
if ( !_Heap_Is_block_in_heap( heap, prev_block ) ) {
_HAssert( false );
return( false );
}
/* As we always coalesce free blocks, the block that preceedes prev_block
must have been used. */
if ( !_Heap_Is_prev_used ( prev_block) ) {
_HAssert( false );
return( false );
}
if ( next_is_free ) { /* coalesce both */
uintptr_t const size = block_size + prev_size + next_block_size;
_Heap_Free_list_remove( next_block );
stats->free_blocks -= 1;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( prev_block, size );
_HAssert(!_Heap_Is_prev_used( next_block));
next_block->prev_size = size;
} else { /* coalesce prev */
uintptr_t const size = block_size + prev_size;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = size;
}
} else if ( next_is_free ) { /* coalesce next */
uintptr_t const size = block_size + next_block_size;
_Heap_Free_list_replace( next_block, block );
block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( block, size );
next_block->prev_size = size;
} else { /* no coalesce */
/* Add 'block' to the head of the free blocks list as it tends to
produce less fragmentation than adding to the tail. */
_Heap_Free_list_insert_after( _Heap_Free_list_head( heap), block );
block->size_and_flag = block_size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = block_size;
/* Statistics */
++stats->free_blocks;
if ( stats->max_free_blocks < stats->free_blocks ) {
stats->max_free_blocks = stats->free_blocks;
}
}
/* Statistics */
--stats->used_blocks;
++stats->frees;
stats->free_size += block_size;
return( true );
}
