/*
* free a malloc_elem block by adding it to the free list. If the
* blocks either immediately before or immediately after newly freed block
* are also free, the blocks are merged together.
*/
int
malloc_elem_free(struct malloc_elem *elem)
{
if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
return -1;
rte_spinlock_lock(&(elem->heap->lock));
struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size);
if (next->state == ELEM_FREE){
/* remove from free list, join to this one */
elem_free_list_remove(next);
join_elem(elem, next);
}
/* check if previous element is free, if so join with it and return,
* need to re-insert in free list, as that element's size is changing
*/
if (elem->prev != NULL && elem->prev->state == ELEM_FREE) {
elem_free_list_remove(elem->prev);
join_elem(elem->prev, elem);
malloc_elem_free_list_insert(elem->prev);
}
/* otherwise add ourselves to the free list */
else {
malloc_elem_free_list_insert(elem);
elem->pad = 0;
}
/* decrease heap's count of allocated elements */
elem->heap->alloc_count--;
rte_spinlock_unlock(&(elem->heap->lock));
return 0;
}
/*
* reserve a block of data in an existing malloc_elem. If the malloc_elem
* is much larger than the data block requested, we split the element in two.
* This function is only called from malloc_heap_alloc so parameter checking
* is not done here, as it's done there previously.
*/
struct malloc_elem *
malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align,
size_t bound)
{
struct malloc_elem *new_elem = elem_start_pt(elem, size, align, bound);
const size_t old_elem_size = (uintptr_t)new_elem - (uintptr_t)elem;
const size_t trailer_size = elem->size - old_elem_size - size -
MALLOC_ELEM_OVERHEAD;
elem_free_list_remove(elem);
if (trailer_size > MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
/* split it, too much free space after elem */
struct malloc_elem *new_free_elem =
RTE_PTR_ADD(new_elem, size + MALLOC_ELEM_OVERHEAD);
split_elem(elem, new_free_elem);
malloc_elem_free_list_insert(new_free_elem);
}
if (old_elem_size < MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
/* don't split it, pad the element instead */
elem->state = ELEM_BUSY;
elem->pad = old_elem_size;
/* put a dummy header in padding, to point to real element header */
if (elem->pad > 0){ /* pad will be at least 64-bytes, as everything
* is cache-line aligned */
new_elem->pad = elem->pad;
new_elem->state = ELEM_PAD;
new_elem->size = elem->size - elem->pad;
set_header(new_elem);
}
return new_elem;
}
/* we are going to split the element in two. The original element
* remains free, and the new element is the one allocated.
* Re-insert original element, in case its new size makes it
* belong on a different list.
*/
split_elem(elem, new_elem);
new_elem->state = ELEM_BUSY;
malloc_elem_free_list_insert(elem);
return new_elem;
}
/*
* Expand the heap with a memseg.
* This reserves the zone and sets a dummy malloc_elem header at the end
* to prevent overflow. The rest of the zone is added to free list as a single
* large free block
*/
static void
malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms)
{
/* allocate the memory block headers, one at end, one at start */
struct malloc_elem *start_elem = (struct malloc_elem *)ms->addr;
struct malloc_elem *end_elem = RTE_PTR_ADD(ms->addr,
ms->len - MALLOC_ELEM_OVERHEAD);
end_elem = RTE_PTR_ALIGN_FLOOR(end_elem, RTE_CACHE_LINE_SIZE);
const size_t elem_size = (uintptr_t)end_elem - (uintptr_t)start_elem;
malloc_elem_init(start_elem, heap, ms, elem_size);
malloc_elem_mkend(end_elem, start_elem);
malloc_elem_free_list_insert(start_elem);
heap->total_size += elem_size;
}
/*
* Expand the heap with a memory area.
*/
static struct malloc_elem *
malloc_heap_add_memory(struct malloc_heap *heap, struct rte_memseg_list *msl,
void *start, size_t len)
{
struct malloc_elem *elem = start;
malloc_elem_init(elem, heap, msl, len);
malloc_elem_insert(elem);
elem = malloc_elem_join_adjacent_free(elem);
malloc_elem_free_list_insert(elem);
return elem;
}
/*
* reserve an extra memory zone and make it available for use by a particular
* heap. This reserves the zone and sets a dummy malloc_elem header at the end
* to prevent overflow. The rest of the zone is added to free list as a single
* large free block
*/
static int
malloc_heap_add_memzone(struct malloc_heap *heap, size_t size, unsigned align)
{
const unsigned mz_flags = 0;
const size_t block_size = get_malloc_memzone_size();
/* ensure the data we want to allocate will fit in the memzone */
const size_t min_size = size + align + MALLOC_ELEM_OVERHEAD * 2;
const struct rte_memzone *mz = NULL;
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
unsigned numa_socket = heap - mcfg->malloc_heaps;
size_t mz_size = min_size;
if (mz_size < block_size)
mz_size = block_size;
char mz_name[RTE_MEMZONE_NAMESIZE];
snprintf(mz_name, sizeof(mz_name), "MALLOC_S%u_HEAP_%u",
numa_socket, heap->mz_count++);
/* try getting a block. if we fail and we don't need as big a block
* as given in the config, we can shrink our request and try again
*/
do {
mz = rte_memzone_reserve(mz_name, mz_size, numa_socket,
mz_flags);
if (mz == NULL)
mz_size /= 2;
} while (mz == NULL && mz_size > min_size);
if (mz == NULL)
return -1;
/* allocate the memory block headers, one at end, one at start */
struct malloc_elem *start_elem = (struct malloc_elem *)mz->addr;
struct malloc_elem *end_elem = RTE_PTR_ADD(mz->addr,
mz_size - MALLOC_ELEM_OVERHEAD);
end_elem = RTE_PTR_ALIGN_FLOOR(end_elem, RTE_CACHE_LINE_SIZE);
const unsigned elem_size = (uintptr_t)end_elem - (uintptr_t)start_elem;
malloc_elem_init(start_elem, heap, mz, elem_size);
malloc_elem_mkend(end_elem, start_elem);
malloc_elem_free_list_insert(start_elem);
/* increase heap total size by size of new memzone */
heap->total_size+=mz_size - MALLOC_ELEM_OVERHEAD;
return 0;
}
/*
* attempt to resize a malloc_elem by expanding into any free space
* immediately after it in memory.
*/
int
malloc_elem_resize(struct malloc_elem *elem, size_t size)
{
const size_t new_size = size + MALLOC_ELEM_OVERHEAD;
/* if we request a smaller size, then always return ok */
const size_t current_size = elem->size - elem->pad;
if (current_size >= new_size)
return 0;
struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size);
rte_spinlock_lock(&elem->heap->lock);
if (next ->state != ELEM_FREE)
goto err_return;
if (current_size + next->size < new_size)
goto err_return;
/* we now know the element fits, so remove from free list,
* join the two
*/
elem_free_list_remove(next);
join_elem(elem, next);
if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD){
/* now we have a big block together. Lets cut it down a bit, by splitting */
struct malloc_elem *split_pt = RTE_PTR_ADD(elem, new_size);
split_pt = RTE_PTR_ALIGN_CEIL(split_pt, RTE_CACHE_LINE_SIZE);
split_elem(elem, split_pt);
malloc_elem_free_list_insert(split_pt);
}
rte_spinlock_unlock(&elem->heap->lock);
return 0;
err_return:
rte_spinlock_unlock(&elem->heap->lock);
return -1;
}
