static Heap_Resize_status _Heap_Resize_block_checked(
Heap_Control *heap,
Heap_Block *block,
uintptr_t alloc_begin,
uintptr_t new_alloc_size,
uintptr_t *old_size,
uintptr_t *new_size
)
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t const block_begin = (uintptr_t) block;
uintptr_t block_size = _Heap_Block_size( block );
uintptr_t block_end = block_begin + block_size;
uintptr_t alloc_size = block_end - alloc_begin + HEAP_ALLOC_BONUS;
Heap_Block *next_block = _Heap_Block_at( block, block_size );
uintptr_t next_block_size = _Heap_Block_size( next_block );
bool next_block_is_free = _Heap_Is_free( next_block );
_HAssert( _Heap_Is_block_in_heap( heap, next_block ) );
_HAssert( _Heap_Is_prev_used( next_block ) );
*old_size = alloc_size;
if ( next_block_is_free ) {
block_size += next_block_size;
alloc_size += next_block_size;
}
if ( new_alloc_size > alloc_size ) {
return HEAP_RESIZE_UNSATISFIED;
}
if ( next_block_is_free ) {
_Heap_Block_set_size( block, block_size );
_Heap_Free_list_remove( next_block );
next_block = _Heap_Block_at( block, block_size );
next_block->size_and_flag |= HEAP_PREV_BLOCK_USED;
/* Statistics */
--stats->free_blocks;
stats->free_size -= next_block_size;
}
block = _Heap_Block_allocate( heap, block, alloc_begin, new_alloc_size );
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
*new_size = (uintptr_t) next_block - alloc_begin + HEAP_ALLOC_BONUS;
/* Statistics */
++stats->resizes;
return HEAP_RESIZE_SUCCESSFUL;
}
bool _Heap_Size_of_alloc_area(
Heap_Control *heap,
void *alloc_begin_ptr,
uintptr_t *alloc_size
)
{
uintptr_t const page_size = heap->page_size;
uintptr_t const alloc_begin = (uintptr_t) alloc_begin_ptr;
Heap_Block *block = _Heap_Block_of_alloc_area( alloc_begin, page_size );
Heap_Block *next_block = NULL;
uintptr_t block_size = 0;
if ( !_Heap_Is_block_in_heap( heap, block ) ) {
return false;
}
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
if (
!_Heap_Is_block_in_heap( heap, next_block )
|| !_Heap_Is_prev_used( next_block )
) {
return false;
}
*alloc_size = (uintptr_t) next_block + HEAP_BLOCK_SIZE_OFFSET - alloc_begin;
return true;
}
static void test_check_free_block(void)
{
Heap_Block *block = NULL;
Heap_Block *next_block = NULL;
Heap_Block *first_free_block = NULL;
Heap_Block *secound_free_block = NULL;
void *p1 = NULL;
puts( "test the _Heap_Walk_check_free_block() function" );
puts( "\tset a previous size for the next block which is not equal to the size of the actual block" );
test_heap_init_default();
block = _Heap_Free_list_first( &TestHeap );
next_block = _Heap_Block_at( block, _Heap_Block_size( block ) );
next_block->prev_size = _Heap_Block_size( block ) - 1;
test_call_heap_walk( false );
puts( "\tclear the previous_used flag of the first free block after an used block" );
test_heap_init_default();
p1 = test_allocate_block();
block = _Heap_Block_of_alloc_area( (uintptr_t) p1, TestHeap.page_size );
first_free_block = _Heap_Free_list_first( &TestHeap );
first_free_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
first_free_block->prev_size = _Heap_Block_size( block );
_Heap_Free_list_insert_after( first_free_block, block );
test_call_heap_walk( false );
puts( "\ttake a free block out of the free list" );
test_heap_init_custom();
test_create_heap_with_gaps();
first_free_block = _Heap_Free_list_first( &TestHeap );
secound_free_block = first_free_block->next;
_Heap_Free_list_remove( secound_free_block );
test_call_heap_walk( false );
}
/*
* Allocate block of size 'alloc_size' from 'the_block' belonging to
* 'the_heap'. Split 'the_block' if possible, otherwise allocate it entirely.
* When split, make the upper part used, and leave the lower part free.
* Return the block allocated.
*
* NOTE: this is similar to _Heap_Block_allocate(), except it makes different
* part of the split block used, and returns address of the block instead of its
* size. We do need such variant for _Heap_Allocate_aligned() as we can't allow
* user pointer to be too far from the beginning of the block, so that we can
* recover start-of-block address from the user pointer without additional
* information stored in the heap.
*/
static
Heap_Block *block_allocate(
Heap_Control *the_heap,
Heap_Block *the_block,
uint32_t alloc_size)
{
Heap_Statistics *const stats = &the_heap->stats;
uint32_t const block_size = _Heap_Block_size(the_block);
uint32_t const the_rest = block_size - alloc_size;
_HAssert(_Heap_Is_aligned(block_size, the_heap->page_size));
_HAssert(_Heap_Is_aligned(alloc_size, the_heap->page_size));
_HAssert(alloc_size <= block_size);
_HAssert(_Heap_Is_prev_used(the_block));
if(the_rest >= the_heap->min_block_size) {
/* Split the block so that lower part is still free, and upper part
becomes used. */
the_block->size = the_rest | HEAP_PREV_USED;
the_block = _Heap_Block_at(the_block, the_rest);
the_block->prev_size = the_rest;
the_block->size = alloc_size;
}
else {
/* Don't split the block as remainder is either zero or too small to be
used as a separate free block. Change 'alloc_size' to the size of the
block and remove the block from the list of free blocks. */
_Heap_Block_remove(the_block);
alloc_size = block_size;
stats->free_blocks -= 1;
}
/* Mark the block as used (in the next block). */
_Heap_Block_at(the_block, alloc_size)->size |= HEAP_PREV_USED;
/* Update statistics */
stats->free_size -= alloc_size;
if(stats->min_free_size > stats->free_size)
stats->min_free_size = stats->free_size;
stats->used_blocks += 1;
return the_block;
}
void _Heap_Get_information(
Heap_Control *the_heap,
Heap_Information_block *the_info
)
{
Heap_Block *the_block = the_heap->first_block;
Heap_Block *const end = the_heap->last_block;
_HAssert(the_block->prev_size == the_heap->page_size);
_HAssert(_Heap_Is_prev_used(the_block));
the_info->Free.number = 0;
the_info->Free.total = 0;
the_info->Free.largest = 0;
the_info->Used.number = 0;
the_info->Used.total = 0;
the_info->Used.largest = 0;
while ( the_block != end ) {
uintptr_t const the_size = _Heap_Block_size(the_block);
Heap_Block *const next_block = _Heap_Block_at(the_block, the_size);
Heap_Information *info;
if ( _Heap_Is_prev_used(next_block) )
info = &the_info->Used;
else
info = &the_info->Free;
info->number++;
info->total += the_size;
if ( info->largest < the_size )
info->largest = the_size;
the_block = next_block;
}
/*
* Handle the last dummy block. Don't consider this block to be
* "used" as client never allocated it. Make 'Used.total' contain this
* blocks' overhead though.
*/
the_info->Used.total += HEAP_BLOCK_HEADER_SIZE;
}
void _Heap_Iterate(
Heap_Control *heap,
Heap_Block_visitor visitor,
void *visitor_arg
)
{
Heap_Block *current = heap->first_block;
Heap_Block *end = heap->last_block;
bool stop = false;
while ( !stop && current != end ) {
uintptr_t size = _Heap_Block_size( current );
Heap_Block *next = _Heap_Block_at( current, size );
bool used = _Heap_Is_prev_used( next );
stop = (*visitor)( current, size, used, visitor_arg );
current = next;
}
}
static void _Heap_Merge_above(
Heap_Control *heap,
Heap_Block *last_block,
uintptr_t extend_area_end
)
{
uintptr_t const page_size = heap->page_size;
uintptr_t const last_block_begin = (uintptr_t) last_block;
uintptr_t const last_block_new_size = _Heap_Align_down(
extend_area_end - last_block_begin - HEAP_BLOCK_HEADER_SIZE,
page_size
);
Heap_Block *const new_last_block =
_Heap_Block_at( last_block, last_block_new_size );
new_last_block->size_and_flag =
(last_block->size_and_flag - last_block_new_size)
| HEAP_PREV_BLOCK_USED;
_Heap_Block_set_size( last_block, last_block_new_size );
_Heap_Free_block( heap, last_block );
}
uintptr_t _Heap_Extend(
Heap_Control *heap,
void *extend_area_begin_ptr,
uintptr_t extend_area_size,
uintptr_t unused __attribute__((unused))
)
{
Heap_Statistics *const stats = &heap->stats;
Heap_Block *const first_block = heap->first_block;
Heap_Block *start_block = first_block;
Heap_Block *merge_below_block = NULL;
Heap_Block *merge_above_block = NULL;
Heap_Block *link_below_block = NULL;
Heap_Block *link_above_block = NULL;
Heap_Block *extend_first_block = NULL;
Heap_Block *extend_last_block = NULL;
uintptr_t const page_size = heap->page_size;
uintptr_t const min_block_size = heap->min_block_size;
uintptr_t const extend_area_begin = (uintptr_t) extend_area_begin_ptr;
uintptr_t const extend_area_end = extend_area_begin + extend_area_size;
uintptr_t const free_size = stats->free_size;
uintptr_t extend_first_block_size = 0;
uintptr_t extended_size = 0;
bool extend_area_ok = false;
if ( extend_area_end < extend_area_begin ) {
return 0;
}
extend_area_ok = _Heap_Get_first_and_last_block(
extend_area_begin,
extend_area_size,
page_size,
min_block_size,
&extend_first_block,
&extend_last_block
);
if (!extend_area_ok ) {
/* For simplicity we reject extend areas that are too small */
return 0;
}
do {
uintptr_t const sub_area_begin = (start_block != first_block) ?
(uintptr_t) start_block : heap->area_begin;
uintptr_t const sub_area_end = start_block->prev_size;
Heap_Block *const end_block =
_Heap_Block_of_alloc_area( sub_area_end, page_size );
if (
sub_area_end > extend_area_begin && extend_area_end > sub_area_begin
) {
return 0;
}
if ( extend_area_end == sub_area_begin ) {
merge_below_block = start_block;
} else if ( extend_area_end < sub_area_end ) {
link_below_block = start_block;
}
if ( sub_area_end == extend_area_begin ) {
start_block->prev_size = extend_area_end;
merge_above_block = end_block;
} else if ( sub_area_end < extend_area_begin ) {
link_above_block = end_block;
}
start_block = _Heap_Block_at( end_block, _Heap_Block_size( end_block ) );
} while ( start_block != first_block );
if ( extend_area_begin < heap->area_begin ) {
heap->area_begin = extend_area_begin;
} else if ( heap->area_end < extend_area_end ) {
heap->area_end = extend_area_end;
}
extend_first_block_size =
(uintptr_t) extend_last_block - (uintptr_t) extend_first_block;
extend_first_block->prev_size = extend_area_end;
extend_first_block->size_and_flag =
extend_first_block_size | HEAP_PREV_BLOCK_USED;
_Heap_Protection_block_initialize( heap, extend_first_block );
extend_last_block->prev_size = extend_first_block_size;
extend_last_block->size_and_flag = 0;
_Heap_Protection_block_initialize( heap, extend_last_block );
if ( (uintptr_t) extend_first_block < (uintptr_t) heap->first_block ) {
heap->first_block = extend_first_block;
} else if ( (uintptr_t) extend_last_block > (uintptr_t) heap->last_block ) {
heap->last_block = extend_last_block;
}
if ( merge_below_block != NULL ) {
_Heap_Merge_below( heap, extend_area_begin, merge_below_block );
} else if ( link_below_block != NULL ) {
_Heap_Link_below(
link_below_block,
extend_last_block
);
}
if ( merge_above_block != NULL ) {
_Heap_Merge_above( heap, merge_above_block, extend_area_end );
} else if ( link_above_block != NULL ) {
_Heap_Link_above(
link_above_block,
extend_first_block,
extend_last_block
);
}
if ( merge_below_block == NULL && merge_above_block == NULL ) {
_Heap_Free_block( heap, extend_first_block );
}
_Heap_Set_last_block_size( heap );
extended_size = stats->free_size - free_size;
/* Statistics */
stats->size += extended_size;
return extended_size;
}
boolean _Heap_Walk(
Heap_Control *the_heap,
int source,
boolean do_dump
)
{
Heap_Block *the_block = the_heap->start;
Heap_Block *const end = the_heap->final;
Heap_Block *const tail = _Heap_Tail(the_heap);
int error = 0;
int passes = 0;
do_dump = FALSE;
/*
* We don't want to allow walking the heap until we have
* transferred control to the user task so we watch the
* system state.
*/
/*
if ( !_System_state_Is_up( _System_state_Get() ) )
return TRUE;
*/
if (source < 0)
source = the_heap->stats.instance;
if (do_dump == TRUE)
printk("\nPASS: %d start %p final %p first %p last %p begin %p end %p\n",
source, the_block, end,
_Heap_First(the_heap), _Heap_Last(the_heap),
the_heap->begin, the_heap->end);
/*
* Handle the 1st block
*/
if (!_Heap_Is_prev_used(the_block)) {
printk("PASS: %d !HEAP_PREV_USED flag of 1st block isn't set\n", source);
error = 1;
}
if (the_block->prev_size != the_heap->page_size) {
printk("PASS: %d !prev_size of 1st block isn't page_size\n", source);
error = 1;
}
while ( the_block != end ) {
uint32_t const the_size = _Heap_Block_size(the_block);
Heap_Block *const next_block = _Heap_Block_at(the_block, the_size);
boolean prev_used = _Heap_Is_prev_used(the_block);
if (do_dump) {
printk("PASS: %d block %p size %d(%c)",
source, the_block, the_size, (prev_used ? 'U' : 'F'));
if (prev_used)
printk(" prev_size %d", the_block->prev_size);
else
printk(" (prev_size) %d", the_block->prev_size);
}
if (!_Heap_Is_block_in(the_heap, next_block)) {
if (do_dump) printk("\n");
printk("PASS: %d !block %p is out of heap\n", source, next_block);
error = 1;
break;
}
if (!_Heap_Is_prev_used(next_block)) {
if (do_dump)
printk( " prev %p next %p", the_block->prev, the_block->next);
if (_Heap_Block_size(the_block) != next_block->prev_size) {
if (do_dump) printk("\n");
printk("PASS: %d !front and back sizes don't match", source);
error = 1;
}
if (!prev_used) {
if (do_dump || error) printk("\n");
printk("PASS: %d !two consecutive blocks are free", source);
error = 1;
}
{ /* Check if 'the_block' is in the free block list */
Heap_Block* block = _Heap_First(the_heap);
while(block != the_block && block != tail)
block = block->next;
if(block != the_block) {
if (do_dump || error) printk("\n");
printk("PASS: %d !the_block not in the free list", source);
error = 1;
}
}
}
if (do_dump || error) printk("\n");
if (the_size < the_heap->min_block_size) {
printk("PASS: %d !block size is too small\n", source);
error = 1;
break;
}
if (!_Heap_Is_aligned( the_size, the_heap->page_size)) {
printk("PASS: %d !block size is misaligned\n", source);
error = 1;
}
if (++passes > (do_dump ? 10 : 0) && error)
break;
the_block = next_block;
}
if (the_block != end) {
printk("PASS: %d !last block address isn't equal to 'final' %p %p\n",
source, the_block, end);
error = 1;
}
if (_Heap_Block_size(the_block) != the_heap->page_size) {
printk("PASS: %d !last block's size isn't page_size (%d != %d)\n", source,
_Heap_Block_size(the_block), the_heap->page_size);
error = 1;
}
if(do_dump && error)
_Internal_error_Occurred( INTERNAL_ERROR_CORE, TRUE, 0xffff0000 );
return error;
}
static void test_main_loop(void)
{
void *p1 = NULL;
void *p2 = NULL;
Heap_Block *block = NULL;
Heap_Block *next_block = NULL;
puts( "Test the main loop" );
puts( "\tset the blocksize so, that the next block is outside the heap" );
test_heap_init_custom();
/* use all blocks */
p1 = test_fill_heap();
block = _Heap_Block_of_alloc_area( (uintptr_t) p1, TestHeap.page_size );
block->size_and_flag = ( 2 * _Heap_Block_size( block ) ) | HEAP_PREV_BLOCK_USED;
test_call_heap_walk( false );
puts( "\twalk a heap with blocks with different states of the previous-used flag" );
test_heap_init_custom();
test_create_heap_with_gaps();
test_allocate_block(); /* fill one gap */
test_call_heap_walk( true );
puts( "\tcreate a block with a not page aligned size" );
test_heap_init_custom();
p1 = test_allocate_block();
p2 = test_allocate_block();
_Heap_Free( &TestHeap, p1 );
block = _Heap_Block_of_alloc_area( (uintptr_t) p2, TestHeap.page_size );
block->size_and_flag = (3 * TestHeap.page_size / 2) & ~HEAP_PREV_BLOCK_USED;
test_call_heap_walk( false );
puts( "\tcreate a block with a size smaller than the min_block_size" );
test_heap_init_default();
p1 = test_allocate_block();
p2 = test_allocate_block();
_Heap_Free( &TestHeap, p1 );
block = _Heap_Block_of_alloc_area( (uintptr_t) p2, TestHeap.page_size );
block->size_and_flag = 0;
test_call_heap_walk( false );
puts( "\tmake a block with a size, so that the block reaches into the next block" );
test_heap_init_default();
p1 = test_allocate_block();
p2 = test_allocate_block();
block = _Heap_Block_of_alloc_area( (uintptr_t) p1, TestHeap.page_size );
block->size_and_flag = ( 3 * _Heap_Block_size( block ) / 2 ) | HEAP_PREV_BLOCK_USED;
test_call_heap_walk( false );
puts( "\tcreate a block with invalid successor" );
test_heap_init_default();
test_allocate_block();
p1 = test_allocate_block();
block = _Heap_Block_of_alloc_area( (uintptr_t) p1, TestHeap.page_size );
block->size_and_flag = (0 - TestHeap.page_size) | HEAP_PREV_BLOCK_USED;
test_call_heap_walk( false );
puts( "\tmake a block with a size, so that it includes the next block" );
test_heap_init_default();
p1 = test_allocate_block();
p2 = test_allocate_block();
block = _Heap_Block_of_alloc_area( (uintptr_t) p1, TestHeap.page_size );
next_block = _Heap_Block_at( block, _Heap_Block_size( block ) );
block->size_and_flag = ( _Heap_Block_size( block ) + _Heap_Block_size( next_block ) ) | HEAP_PREV_BLOCK_USED;
test_call_heap_walk( true );
}
void _Heap_Block_split(
Heap_Control *heap,
Heap_Block *block,
Heap_Block *free_list_anchor,
uintptr_t alloc_size
)
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t const page_size = heap->page_size;
uintptr_t const min_block_size = heap->min_block_size;
uintptr_t const min_alloc_size = min_block_size - HEAP_BLOCK_HEADER_SIZE;
uintptr_t const block_size = _Heap_Block_size( block );
uintptr_t const used_size =
_Heap_Max( alloc_size, min_alloc_size ) + HEAP_BLOCK_HEADER_SIZE;
uintptr_t const used_block_size = _Heap_Align_up( used_size, page_size );
uintptr_t const free_size = block_size + HEAP_BLOCK_SIZE_OFFSET - used_size;
uintptr_t const free_size_limit = min_block_size + HEAP_BLOCK_SIZE_OFFSET;
Heap_Block *next_block = _Heap_Block_at( block, block_size );
_HAssert( used_size <= block_size + HEAP_BLOCK_SIZE_OFFSET );
_HAssert( used_size + free_size == block_size + HEAP_BLOCK_SIZE_OFFSET );
if ( free_size >= free_size_limit ) {
Heap_Block *const free_block = _Heap_Block_at( block, used_block_size );
uintptr_t free_block_size = block_size - used_block_size;
_HAssert( used_block_size + free_block_size == block_size );
_Heap_Block_set_size( block, used_block_size );
/* Statistics */
stats->free_size += free_block_size;
if ( _Heap_Is_used( next_block ) ) {
_Heap_Free_list_insert_after( free_list_anchor, free_block );
/* Statistics */
++stats->free_blocks;
} else {
uintptr_t const next_block_size = _Heap_Block_size( next_block );
_Heap_Free_list_replace( next_block, free_block );
free_block_size += next_block_size;
next_block = _Heap_Block_at( free_block, free_block_size );
}
free_block->size_and_flag = free_block_size | HEAP_PREV_BLOCK_USED;
next_block->prev_size = free_block_size;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
} else {
next_block->size_and_flag |= HEAP_PREV_BLOCK_USED;
}
}
bool _Heap_Walk(
Heap_Control *heap,
int source,
bool dump
)
{
uintptr_t const page_size = heap->page_size;
uintptr_t const min_block_size = heap->min_block_size;
Heap_Block *const first_block = heap->first_block;
Heap_Block *const last_block = heap->last_block;
Heap_Block *block = first_block;
Heap_Walk_printer printer = dump ?
_Heap_Walk_print : _Heap_Walk_print_nothing;
if ( !_System_state_Is_up( _System_state_Get() ) ) {
return true;
}
if ( !_Heap_Walk_check_control( source, printer, heap ) ) {
return false;
}
do {
uintptr_t const block_begin = (uintptr_t) block;
uintptr_t const block_size = _Heap_Block_size( block );
bool const prev_used = _Heap_Is_prev_used( block );
Heap_Block *const next_block = _Heap_Block_at( block, block_size );
uintptr_t const next_block_begin = (uintptr_t) next_block;
bool const is_not_last_block = block != last_block;
if ( !_Heap_Is_block_in_heap( heap, next_block ) ) {
(*printer)(
source,
true,
"block 0x%08x: next block 0x%08x not in heap\n",
block,
next_block
);
return false;
}
if ( !_Heap_Is_aligned( block_size, page_size ) && is_not_last_block ) {
(*printer)(
source,
true,
"block 0x%08x: block size %u not page aligned\n",
block,
block_size
);
return false;
}
if ( block_size < min_block_size && is_not_last_block ) {
(*printer)(
source,
true,
"block 0x%08x: size %u < min block size %u\n",
block,
block_size,
min_block_size
);
return false;
}
if ( next_block_begin <= block_begin && is_not_last_block ) {
(*printer)(
source,
true,
"block 0x%08x: next block 0x%08x is not a successor\n",
block,
next_block
);
return false;
}
if ( !_Heap_Is_prev_used( next_block ) ) {
if ( !_Heap_Walk_check_free_block( source, printer, heap, block ) ) {
return false;
}
} else if (prev_used) {
(*printer)(
source,
false,
"block 0x%08x: size %u\n",
block,
block_size
);
} else {
(*printer)(
source,
false,
"block 0x%08x: size %u, prev_size %u\n",
block,
block_size,
block->prev_size
);
}
block = next_block;
} while ( block != first_block );
return true;
}
static bool _Heap_Walk_check_free_block(
int source,
Heap_Walk_printer printer,
Heap_Control *heap,
Heap_Block *block
)
{
Heap_Block *const free_list_tail = _Heap_Free_list_tail( heap );
Heap_Block *const free_list_head = _Heap_Free_list_head( heap );
Heap_Block *const first_free_block = _Heap_Free_list_first( heap );
Heap_Block *const last_free_block = _Heap_Free_list_last( heap );
bool const prev_used = _Heap_Is_prev_used( block );
uintptr_t const block_size = _Heap_Block_size( block );
Heap_Block *const next_block = _Heap_Block_at( block, block_size );
(*printer)(
source,
false,
"block 0x%08x: size %u, prev 0x%08x%s, next 0x%08x%s\n",
block,
block_size,
block->prev,
block->prev == first_free_block ?
" (= first free)"
: (block->prev == free_list_head ? " (= head)" : ""),
block->next,
block->next == last_free_block ?
" (= last free)"
: (block->next == free_list_tail ? " (= tail)" : "")
);
if ( block_size != next_block->prev_size ) {
(*printer)(
source,
true,
"block 0x%08x: size %u != size %u (in next block 0x%08x)\n",
block,
block_size,
next_block->prev_size,
next_block
);
return false;
}
if ( !prev_used ) {
(*printer)(
source,
true,
"block 0x%08x: two consecutive blocks are free\n",
block
);
return false;
}
if ( !_Heap_Walk_is_in_free_list( heap, block ) ) {
(*printer)(
source,
true,
"block 0x%08x: free block not in free list\n",
block
);
return false;
}
return true;
}
static bool _Heap_Walk_check_control(
int source,
Heap_Walk_printer printer,
Heap_Control *heap
)
{
uintptr_t const page_size = heap->page_size;
uintptr_t const min_block_size = heap->min_block_size;
Heap_Block *const first_free_block = _Heap_Free_list_first( heap );
Heap_Block *const last_free_block = _Heap_Free_list_last( heap );
Heap_Block *const first_block = heap->first_block;
Heap_Block *const last_block = heap->last_block;
(*printer)(
source,
false,
"page size %u, min block size %u\n"
"\tarea begin 0x%08x, area end 0x%08x\n"
"\tfirst block 0x%08x, last block 0x%08x\n"
"\tfirst free 0x%08x, last free 0x%08x\n",
page_size, min_block_size,
heap->area_begin, heap->area_end,
first_block, last_block,
first_free_block, last_free_block
);
if ( page_size == 0 ) {
(*printer)( source, true, "page size is zero\n" );
return false;
}
if ( !_Addresses_Is_aligned( (void *) page_size ) ) {
(*printer)(
source,
true,
"page size %u not CPU aligned\n",
page_size
);
return false;
}
if ( !_Heap_Is_aligned( min_block_size, page_size ) ) {
(*printer)(
source,
true,
"min block size %u not page aligned\n",
min_block_size
);
return false;
}
if (
!_Heap_Is_aligned( _Heap_Alloc_area_of_block( first_block ), page_size )
) {
(*printer)(
source,
true,
"first block 0x%08x: alloc area not page aligned\n",
first_block
);
return false;
}
if ( !_Heap_Is_prev_used( first_block ) ) {
(*printer)(
source,
true,
"first block: HEAP_PREV_BLOCK_USED is cleared\n"
);
return false;
}
if ( _Heap_Is_free( last_block ) ) {
(*printer)(
source,
true,
"last block: is free\n"
);
return false;
}
if (
_Heap_Block_at( last_block, _Heap_Block_size( last_block ) ) != first_block
) {
(*printer)(
source,
true,
"last block: next block is not the first block\n"
);
return false;
}
return _Heap_Walk_check_free_list( source, printer, heap );
}
bool _Heap_Free( Heap_Control *heap, void *alloc_begin_ptr )
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t alloc_begin;
Heap_Block *block;
Heap_Block *next_block = NULL;
uintptr_t block_size = 0;
uintptr_t next_block_size = 0;
bool next_is_free = false;
/*
* If NULL return true so a free on NULL is considered a valid release. This
* is a special case that could be handled by the in heap check how-ever that
* would result in false being returned which is wrong.
*/
if ( alloc_begin_ptr == NULL ) {
return true;
}
alloc_begin = (uintptr_t) alloc_begin_ptr;
block = _Heap_Block_of_alloc_area( alloc_begin, heap->page_size );
if ( !_Heap_Is_block_in_heap( heap, block ) ) {
return false;
}
_Heap_Protection_block_check( heap, block );
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
if ( !_Heap_Is_block_in_heap( heap, next_block ) ) {
return false;
}
_Heap_Protection_block_check( heap, next_block );
if ( !_Heap_Is_prev_used( next_block ) ) {
_Heap_Protection_block_error( heap, block );
return false;
}
if ( !_Heap_Protection_determine_block_free( heap, block ) ) {
return true;
}
next_block_size = _Heap_Block_size( next_block );
next_is_free = next_block != heap->last_block
&& !_Heap_Is_prev_used( _Heap_Block_at( next_block, next_block_size ));
if ( !_Heap_Is_prev_used( block ) ) {
uintptr_t const prev_size = block->prev_size;
Heap_Block * const prev_block = _Heap_Block_at( block, -prev_size );
if ( !_Heap_Is_block_in_heap( heap, prev_block ) ) {
_HAssert( false );
return( false );
}
/* As we always coalesce free blocks, the block that preceedes prev_block
must have been used. */
if ( !_Heap_Is_prev_used ( prev_block) ) {
_HAssert( false );
return( false );
}
if ( next_is_free ) { /* coalesce both */
uintptr_t const size = block_size + prev_size + next_block_size;
_Heap_Free_list_remove( next_block );
stats->free_blocks -= 1;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( prev_block, size );
_HAssert(!_Heap_Is_prev_used( next_block));
next_block->prev_size = size;
} else { /* coalesce prev */
uintptr_t const size = block_size + prev_size;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = size;
}
} else if ( next_is_free ) { /* coalesce next */
uintptr_t const size = block_size + next_block_size;
_Heap_Free_list_replace( next_block, block );
block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( block, size );
next_block->prev_size = size;
} else { /* no coalesce */
/* Add 'block' to the head of the free blocks list as it tends to
produce less fragmentation than adding to the tail. */
_Heap_Free_list_insert_after( _Heap_Free_list_head( heap), block );
block->size_and_flag = block_size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = block_size;
/* Statistics */
++stats->free_blocks;
if ( stats->max_free_blocks < stats->free_blocks ) {
stats->max_free_blocks = stats->free_blocks;
}
}
/* Statistics */
--stats->used_blocks;
++stats->frees;
stats->free_size += block_size;
return( true );
}
Heap_Extend_status _Heap_Extend(
Heap_Control *heap,
void *area_begin_ptr,
uintptr_t area_size,
uintptr_t *amount_extended
)
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t const area_begin = (uintptr_t) area_begin_ptr;
uintptr_t const heap_area_begin = heap->area_begin;
uintptr_t const heap_area_end = heap->area_end;
uintptr_t const new_heap_area_end = heap_area_end + area_size;
uintptr_t extend_size = 0;
Heap_Block *const last_block = heap->last_block;
/*
* There are five possibilities for the location of starting
* address:
*
* 1. non-contiguous lower address (NOT SUPPORTED)
* 2. contiguous lower address (NOT SUPPORTED)
* 3. in the heap (ERROR)
* 4. contiguous higher address (SUPPORTED)
* 5. non-contiguous higher address (NOT SUPPORTED)
*
* As noted, this code only supports (4).
*/
if ( area_begin >= heap_area_begin && area_begin < heap_area_end ) {
return HEAP_EXTEND_ERROR; /* case 3 */
} else if ( area_begin != heap_area_end ) {
return HEAP_EXTEND_NOT_IMPLEMENTED; /* cases 1, 2, and 5 */
}
/*
* Currently only case 4 should make it to this point.
* The basic trick is to make the extend area look like a used
* block and free it.
*/
heap->area_end = new_heap_area_end;
extend_size = new_heap_area_end
- (uintptr_t) last_block - HEAP_BLOCK_HEADER_SIZE;
extend_size = _Heap_Align_down( extend_size, heap->page_size );
*amount_extended = extend_size;
if( extend_size >= heap->min_block_size ) {
Heap_Block *const new_last_block = _Heap_Block_at( last_block, extend_size );
_Heap_Block_set_size( last_block, extend_size );
new_last_block->size_and_flag =
((uintptr_t) heap->first_block - (uintptr_t) new_last_block)
| HEAP_PREV_BLOCK_USED;
heap->last_block = new_last_block;
/* Statistics */
stats->size += extend_size;
++stats->used_blocks;
--stats->frees; /* Do not count subsequent call as actual free() */
_Heap_Free( heap, (void *) _Heap_Alloc_area_of_block( last_block ));
}
return HEAP_EXTEND_SUCCESSFUL;
}
bool _Heap_Free( Heap_Control *heap, void *alloc_begin_ptr )
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t alloc_begin = (uintptr_t) alloc_begin_ptr;
Heap_Block *block =
_Heap_Block_of_alloc_area( alloc_begin, heap->page_size );
Heap_Block *next_block = NULL;
uintptr_t block_size = 0;
uintptr_t next_block_size = 0;
bool next_is_free = false;
if ( !_Heap_Is_block_in_heap( heap, block ) ) {
return false;
}
block_size = _Heap_Block_size( block );
next_block = _Heap_Block_at( block, block_size );
if ( !_Heap_Is_block_in_heap( heap, next_block ) ) {
_HAssert( false );
return false;
}
if ( !_Heap_Is_prev_used( next_block ) ) {
_HAssert( false );
return false;
}
next_block_size = _Heap_Block_size( next_block );
next_is_free = next_block != heap->last_block
&& !_Heap_Is_prev_used( _Heap_Block_at( next_block, next_block_size ));
if ( !_Heap_Is_prev_used( block ) ) {
uintptr_t const prev_size = block->prev_size;
Heap_Block * const prev_block = _Heap_Block_at( block, -prev_size );
if ( !_Heap_Is_block_in_heap( heap, prev_block ) ) {
_HAssert( false );
return( false );
}
/* As we always coalesce free blocks, the block that preceedes prev_block
must have been used. */
if ( !_Heap_Is_prev_used ( prev_block) ) {
_HAssert( false );
return( false );
}
if ( next_is_free ) { /* coalesce both */
uintptr_t const size = block_size + prev_size + next_block_size;
_Heap_Free_list_remove( next_block );
stats->free_blocks -= 1;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( prev_block, size );
_HAssert(!_Heap_Is_prev_used( next_block));
next_block->prev_size = size;
} else { /* coalesce prev */
uintptr_t const size = block_size + prev_size;
prev_block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = size;
}
} else if ( next_is_free ) { /* coalesce next */
uintptr_t const size = block_size + next_block_size;
_Heap_Free_list_replace( next_block, block );
block->size_and_flag = size | HEAP_PREV_BLOCK_USED;
next_block = _Heap_Block_at( block, size );
next_block->prev_size = size;
} else { /* no coalesce */
/* Add 'block' to the head of the free blocks list as it tends to
produce less fragmentation than adding to the tail. */
_Heap_Free_list_insert_after( _Heap_Free_list_head( heap), block );
block->size_and_flag = block_size | HEAP_PREV_BLOCK_USED;
next_block->size_and_flag &= ~HEAP_PREV_BLOCK_USED;
next_block->prev_size = block_size;
/* Statistics */
++stats->free_blocks;
if ( stats->max_free_blocks < stats->free_blocks ) {
stats->max_free_blocks = stats->free_blocks;
}
}
/* Statistics */
--stats->used_blocks;
++stats->frees;
stats->free_size += block_size;
return( true );
}
uintptr_t _Heap_Initialize(
Heap_Control *heap,
void *heap_area_begin_ptr,
uintptr_t heap_area_size,
uintptr_t page_size
)
{
Heap_Statistics *const stats = &heap->stats;
uintptr_t const heap_area_begin = (uintptr_t) heap_area_begin_ptr;
uintptr_t const heap_area_end = heap_area_begin + heap_area_size;
uintptr_t alloc_area_begin = heap_area_begin + HEAP_BLOCK_HEADER_SIZE;
uintptr_t alloc_area_size = 0;
uintptr_t first_block_begin = 0;
uintptr_t first_block_size = 0;
uintptr_t min_block_size = 0;
uintptr_t overhead = 0;
Heap_Block *first_block = NULL;
Heap_Block *last_block = NULL;
if ( page_size == 0 ) {
page_size = CPU_ALIGNMENT;
} else {
page_size = _Heap_Align_up( page_size, CPU_ALIGNMENT );
if ( page_size < CPU_ALIGNMENT ) {
/* Integer overflow */
return 0;
}
}
min_block_size = _Heap_Align_up( sizeof( Heap_Block ), page_size );
alloc_area_begin = _Heap_Align_up( alloc_area_begin, page_size );
first_block_begin = alloc_area_begin - HEAP_BLOCK_HEADER_SIZE;
overhead = HEAP_BLOCK_HEADER_SIZE + (first_block_begin - heap_area_begin);
first_block_size = heap_area_size - overhead;
first_block_size = _Heap_Align_down ( first_block_size, page_size );
alloc_area_size = first_block_size - HEAP_BLOCK_HEADER_SIZE;
if (
heap_area_end < heap_area_begin
|| heap_area_size <= overhead
|| first_block_size < min_block_size
) {
/* Invalid area or area too small */
return 0;
}
/* First block */
first_block = (Heap_Block *) first_block_begin;
first_block->prev_size = page_size;
first_block->size_and_flag = first_block_size | HEAP_PREV_BLOCK_USED;
first_block->next = _Heap_Free_list_tail( heap );
first_block->prev = _Heap_Free_list_head( heap );
/*
* Last block.
*
* The next block of the last block is the first block. Since the first
* block indicates that the previous block is used, this ensures that the
* last block appears as used for the _Heap_Is_used() and _Heap_Is_free()
* functions.
*/
last_block = _Heap_Block_at( first_block, first_block_size );
last_block->prev_size = first_block_size;
last_block->size_and_flag = first_block_begin - (uintptr_t) last_block;
/* Heap control */
heap->page_size = page_size;
heap->min_block_size = min_block_size;
heap->area_begin = heap_area_begin;
heap->area_end = heap_area_end;
heap->first_block = first_block;
heap->last_block = last_block;
_Heap_Free_list_head( heap )->next = first_block;
_Heap_Free_list_tail( heap )->prev = first_block;
/* Statistics */
stats->size = first_block_size;
stats->free_size = first_block_size;
stats->min_free_size = first_block_size;
stats->free_blocks = 1;
stats->max_free_blocks = 1;
stats->used_blocks = 0;
stats->max_search = 0;
stats->allocs = 0;
stats->searches = 0;
stats->frees = 0;
stats->resizes = 0;
stats->instance = instance++;
_HAssert( _Heap_Is_aligned( heap->page_size, CPU_ALIGNMENT ) );
_HAssert( _Heap_Is_aligned( heap->min_block_size, page_size ) );
_HAssert(
_Heap_Is_aligned( _Heap_Alloc_area_of_block( first_block ), page_size )
);
_HAssert(
_Heap_Is_aligned( _Heap_Alloc_area_of_block( last_block ), page_size )
);
return alloc_area_size;
}
