JEMALLOC_INLINE_C int
extent_ad_comp(const extent_node_t *a, const extent_node_t *b)
{
vaddr_t a_addr = (vaddr_t)extent_node_addr_get(a);
vaddr_t b_addr = (vaddr_t)extent_node_addr_get(b);
return ((a_addr > b_addr) - (a_addr < b_addr));
}
JEMALLOC_INLINE_C int
extent_ad_comp(extent_node_t *a, extent_node_t *b)
{
uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a);
uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b);
return ((a_addr > b_addr) - (a_addr < b_addr));
}
static bool
huge_node_set(const void *ptr, extent_node_t *node)
{
assert(extent_node_addr_get(node) == ptr);
assert(!extent_node_achunk_get(node));
return (chunk_register(ptr, node));
}
/* je_iterate calls callback for each allocation found in the memory region
* between [base, base+size). base will be rounded down to by the jemalloc
* chunk size, and base+size will be rounded up to the chunk size. If no memory
* managed by jemalloc is found in the requested region, je_iterate returns -1
* and sets errno to EINVAL.
*
* je_iterate must be called when no allocations are in progress, either
* when single-threaded (for example just after a fork), or between
* jemalloc_prefork() and jemalloc_postfork_parent(). The callback must
* not attempt to allocate with jemalloc.
*/
int je_iterate(uintptr_t base, size_t size,
void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) {
int error = EINVAL;
uintptr_t ptr = (uintptr_t)CHUNK_ADDR2BASE(base);
uintptr_t end = CHUNK_CEILING(base + size);
while (ptr < end) {
extent_node_t *node;
node = chunk_lookup((void *)ptr, false);
if (node == NULL) {
ptr += chunksize;
continue;
}
assert(extent_node_achunk_get(node) ||
(uintptr_t)extent_node_addr_get(node) == ptr);
error = 0;
if (extent_node_achunk_get(node)) {
/* Chunk */
arena_chunk_t *chunk = (arena_chunk_t *)ptr;
ptr += chunksize;
if (&chunk->node != node) {
/* Empty retained chunk */
continue;
}
je_iterate_chunk(chunk, callback, arg);
} else if ((uintptr_t)extent_node_addr_get(node) == ptr) {
/* Huge allocation */
callback(ptr, extent_node_size_get(node), arg);
ptr += extent_node_size_get(node);
}
}
if (error) {
set_errno(error);
return -1;
}
return 0;
}
void
huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
{
extent_node_t *node;
arena_t *arena;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
huge_node_unset(ptr, node);
malloc_mutex_lock(&arena->huge_mtx);
ql_remove(&arena->huge, node, ql_link);
malloc_mutex_unlock(&arena->huge_mtx);
huge_dalloc_junk(extent_node_addr_get(node),
extent_node_size_get(node));
arena_chunk_dalloc_huge(extent_node_arena_get(node),
extent_node_addr_get(node), extent_node_size_get(node));
idalloctm(tsd, node, tcache, true, true);
}
JEMALLOC_INLINE_C int
extent_szad_comp(const extent_node_t *a, const extent_node_t *b)
{
int ret;
size_t a_qsize = extent_quantize(extent_node_size_get(a));
size_t b_qsize = extent_quantize(extent_node_size_get(b));
/*
* Compare based on quantized size rather than size, in order to sort
* equally useful extents only by address.
*/
ret = (a_qsize > b_qsize) - (a_qsize < b_qsize);
if (ret == 0) {
uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a);
uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b);
ret = (a_addr > b_addr) - (a_addr < b_addr);
}
return (ret);
}
/*
* base_alloc() guarantees demand-zeroed memory, in order to make multi-page
* sparse data structures such as radix tree nodes efficient with respect to
* physical memory usage.
*/
void *
base_alloc(tsdn_t *tsdn, size_t size)
{
void *ret;
size_t csize, usize;
extent_node_t *node;
extent_node_t key;
/*
* Round size up to nearest multiple of the cacheline size, so that
* there is no chance of false cache line sharing.
*/
csize = CACHELINE_CEILING(size);
usize = s2u(csize);
extent_node_init(&key, NULL, NULL, usize, 0, false, false);
malloc_mutex_lock(tsdn, &base_mtx);
node = extent_tree_szsnad_nsearch(&base_avail_szsnad, &key);
if (node != NULL) {
/* Use existing space. */
extent_tree_szsnad_remove(&base_avail_szsnad, node);
} else {
/* Try to allocate more space. */
node = base_chunk_alloc(tsdn, csize);
}
if (node == NULL) {
ret = NULL;
goto label_return;
}
ret = extent_node_addr_get(node);
if (extent_node_size_get(node) > csize) {
extent_node_addr_set(node, (void *)((uintptr_t)ret + csize));
extent_node_size_set(node, extent_node_size_get(node) - csize);
extent_tree_szsnad_insert(&base_avail_szsnad, node);
} else
base_node_dalloc(tsdn, node);
if (config_stats) {
base_allocated += csize;
/*
* Add one PAGE to base_resident for every page boundary that is
* crossed by the new allocation.
*/
base_resident += PAGE_CEILING((vaddr_t)ret + csize) -
PAGE_CEILING((vaddr_t)ret);
}
JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize);
label_return:
malloc_mutex_unlock(tsdn, &base_mtx);
return (ret);
}
