/* * free a malloc_elem block by adding it to the free list. If the * blocks either immediately before or immediately after newly freed block * are also free, the blocks are merged together. */ int malloc_elem_free(struct malloc_elem *elem) { if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY) return -1; rte_spinlock_lock(&(elem->heap->lock)); struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size); if (next->state == ELEM_FREE){ /* remove from free list, join to this one */ elem_free_list_remove(next); join_elem(elem, next); } /* check if previous element is free, if so join with it and return, * need to re-insert in free list, as that element's size is changing */ if (elem->prev != NULL && elem->prev->state == ELEM_FREE) { elem_free_list_remove(elem->prev); join_elem(elem->prev, elem); malloc_elem_free_list_insert(elem->prev); } /* otherwise add ourselves to the free list */ else { malloc_elem_free_list_insert(elem); elem->pad = 0; } /* decrease heap's count of allocated elements */ elem->heap->alloc_count--; rte_spinlock_unlock(&(elem->heap->lock)); return 0; }
/* * reserve a block of data in an existing malloc_elem. If the malloc_elem * is much larger than the data block requested, we split the element in two. * This function is only called from malloc_heap_alloc so parameter checking * is not done here, as it's done there previously. */ struct malloc_elem * malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align, size_t bound) { struct malloc_elem *new_elem = elem_start_pt(elem, size, align, bound); const size_t old_elem_size = (uintptr_t)new_elem - (uintptr_t)elem; const size_t trailer_size = elem->size - old_elem_size - size - MALLOC_ELEM_OVERHEAD; elem_free_list_remove(elem); if (trailer_size > MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) { /* split it, too much free space after elem */ struct malloc_elem *new_free_elem = RTE_PTR_ADD(new_elem, size + MALLOC_ELEM_OVERHEAD); split_elem(elem, new_free_elem); malloc_elem_free_list_insert(new_free_elem); } if (old_elem_size < MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) { /* don't split it, pad the element instead */ elem->state = ELEM_BUSY; elem->pad = old_elem_size; /* put a dummy header in padding, to point to real element header */ if (elem->pad > 0){ /* pad will be at least 64-bytes, as everything * is cache-line aligned */ new_elem->pad = elem->pad; new_elem->state = ELEM_PAD; new_elem->size = elem->size - elem->pad; set_header(new_elem); } return new_elem; } /* we are going to split the element in two. The original element * remains free, and the new element is the one allocated. * Re-insert original element, in case its new size makes it * belong on a different list. */ split_elem(elem, new_elem); new_elem->state = ELEM_BUSY; malloc_elem_free_list_insert(elem); return new_elem; }
/* * Expand the heap with a memseg. * This reserves the zone and sets a dummy malloc_elem header at the end * to prevent overflow. The rest of the zone is added to free list as a single * large free block */ static void malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms) { /* allocate the memory block headers, one at end, one at start */ struct malloc_elem *start_elem = (struct malloc_elem *)ms->addr; struct malloc_elem *end_elem = RTE_PTR_ADD(ms->addr, ms->len - MALLOC_ELEM_OVERHEAD); end_elem = RTE_PTR_ALIGN_FLOOR(end_elem, RTE_CACHE_LINE_SIZE); const size_t elem_size = (uintptr_t)end_elem - (uintptr_t)start_elem; malloc_elem_init(start_elem, heap, ms, elem_size); malloc_elem_mkend(end_elem, start_elem); malloc_elem_free_list_insert(start_elem); heap->total_size += elem_size; }
/* * Expand the heap with a memory area. */ static struct malloc_elem * malloc_heap_add_memory(struct malloc_heap *heap, struct rte_memseg_list *msl, void *start, size_t len) { struct malloc_elem *elem = start; malloc_elem_init(elem, heap, msl, len); malloc_elem_insert(elem); elem = malloc_elem_join_adjacent_free(elem); malloc_elem_free_list_insert(elem); return elem; }
/* * reserve an extra memory zone and make it available for use by a particular * heap. This reserves the zone and sets a dummy malloc_elem header at the end * to prevent overflow. The rest of the zone is added to free list as a single * large free block */ static int malloc_heap_add_memzone(struct malloc_heap *heap, size_t size, unsigned align) { const unsigned mz_flags = 0; const size_t block_size = get_malloc_memzone_size(); /* ensure the data we want to allocate will fit in the memzone */ const size_t min_size = size + align + MALLOC_ELEM_OVERHEAD * 2; const struct rte_memzone *mz = NULL; struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; unsigned numa_socket = heap - mcfg->malloc_heaps; size_t mz_size = min_size; if (mz_size < block_size) mz_size = block_size; char mz_name[RTE_MEMZONE_NAMESIZE]; snprintf(mz_name, sizeof(mz_name), "MALLOC_S%u_HEAP_%u", numa_socket, heap->mz_count++); /* try getting a block. if we fail and we don't need as big a block * as given in the config, we can shrink our request and try again */ do { mz = rte_memzone_reserve(mz_name, mz_size, numa_socket, mz_flags); if (mz == NULL) mz_size /= 2; } while (mz == NULL && mz_size > min_size); if (mz == NULL) return -1; /* allocate the memory block headers, one at end, one at start */ struct malloc_elem *start_elem = (struct malloc_elem *)mz->addr; struct malloc_elem *end_elem = RTE_PTR_ADD(mz->addr, mz_size - MALLOC_ELEM_OVERHEAD); end_elem = RTE_PTR_ALIGN_FLOOR(end_elem, RTE_CACHE_LINE_SIZE); const unsigned elem_size = (uintptr_t)end_elem - (uintptr_t)start_elem; malloc_elem_init(start_elem, heap, mz, elem_size); malloc_elem_mkend(end_elem, start_elem); malloc_elem_free_list_insert(start_elem); /* increase heap total size by size of new memzone */ heap->total_size+=mz_size - MALLOC_ELEM_OVERHEAD; return 0; }
/* * attempt to resize a malloc_elem by expanding into any free space * immediately after it in memory. */ int malloc_elem_resize(struct malloc_elem *elem, size_t size) { const size_t new_size = size + MALLOC_ELEM_OVERHEAD; /* if we request a smaller size, then always return ok */ const size_t current_size = elem->size - elem->pad; if (current_size >= new_size) return 0; struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size); rte_spinlock_lock(&elem->heap->lock); if (next ->state != ELEM_FREE) goto err_return; if (current_size + next->size < new_size) goto err_return; /* we now know the element fits, so remove from free list, * join the two */ elem_free_list_remove(next); join_elem(elem, next); if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD){ /* now we have a big block together. Lets cut it down a bit, by splitting */ struct malloc_elem *split_pt = RTE_PTR_ADD(elem, new_size); split_pt = RTE_PTR_ALIGN_CEIL(split_pt, RTE_CACHE_LINE_SIZE); split_elem(elem, split_pt); malloc_elem_free_list_insert(split_pt); } rte_spinlock_unlock(&elem->heap->lock); return 0; err_return: rte_spinlock_unlock(&elem->heap->lock); return -1; }