/* Lock and return an arena that can be reused for memory allocation.
Avoid AVOID_ARENA as we have already failed to allocate memory in
it and it is currently locked. */
static mstate
reused_arena (mstate avoid_arena)
{
mstate result;
static mstate next_to_use;
if (next_to_use == NULL)
next_to_use = &main_arena;
result = next_to_use;
do
{
if (!mutex_trylock(&result->mutex))
goto out;
result = result->next;
}
while (result != next_to_use);
/* Avoid AVOID_ARENA as we have already failed to allocate memory
in that arena and it is currently locked. */
if (result == avoid_arena)
result = result->next;
/* No arena available. Wait for the next in line. */
(void)mutex_lock(&result->mutex);
out:
tsd_setspecific(arena_key, (void *)result);
THREAD_STAT(++(result->stat_lock_loop));
next_to_use = result->next;
return result;
}
static mstate
reused_arena (void)
{
mstate result;
static mstate next_to_use;
if (next_to_use == NULL)
next_to_use = &main_arena;
result = next_to_use;
do
{
if (!mutex_trylock(&result->mutex))
goto out;
result = result->next;
}
while (result != next_to_use);
/* No arena available. Wait for the next in line. */
(void)mutex_lock(&result->mutex);
out:
tsd_setspecific(arena_key, (void *)result);
THREAD_STAT(++(result->stat_lock_loop));
next_to_use = result->next;
return result;
}
static void*
memalign_starter(size_t align, size_t sz, const void *caller)
{
void* victim;
/*ptmalloc_init_minimal();*/
victim = mspace_memalign(arena_to_mspace(&main_arena), align, sz);
THREAD_STAT(++main_arena.stat_starter);
return victim;
}
static mstate
_int_new_arena(size_t size)
{
mstate a;
heap_info *h;
char *ptr;
unsigned long misalign;
h = new_heap(size + (sizeof(*h) + sizeof(*a) + MALLOC_ALIGNMENT),
mp_.top_pad);
if(!h) {
/* Maybe size is too large to fit in a single heap. So, just try
to create a minimally-sized arena and let _int_malloc() attempt
to deal with the large request via mmap_chunk(). */
h = new_heap(sizeof(*h) + sizeof(*a) + MALLOC_ALIGNMENT, mp_.top_pad);
if(!h)
return 0;
}
a = h->ar_ptr = (mstate)(h+1);
malloc_init_state(a);
/*a->next = NULL;*/
a->system_mem = a->max_system_mem = h->size;
arena_mem += h->size;
/* Set up the top chunk, with proper alignment. */
ptr = (char *)(a + 1);
misalign = (unsigned long)chunk2mem(ptr) & MALLOC_ALIGN_MASK;
if (misalign > 0)
ptr += MALLOC_ALIGNMENT - misalign;
top(a) = (mchunkptr)ptr;
set_head(top(a), (((char*)h + h->size) - ptr) | PREV_INUSE);
tsd_setspecific(arena_key, (void *)a);
mutex_init(&a->mutex);
(void)mutex_lock(&a->mutex);
#ifdef PER_THREAD
(void)mutex_lock(&list_lock);
#endif
/* Add the new arena to the global list. */
a->next = main_arena.next;
atomic_write_barrier ();
main_arena.next = a;
#ifdef PER_THREAD
(void)mutex_unlock(&list_lock);
#endif
THREAD_STAT(++(a->stat_lock_loop));
return a;
}
static void
free_starter(void* mem, const void *caller)
{
if (mem) {
mchunkptr p = mem2chunk(mem);
void *msp = arena_to_mspace(&main_arena);
if (is_mmapped(p))
munmap_chunk(msp, p);
else
mspace_free(msp, mem);
}
THREAD_STAT(++main_arena.stat_starter);
}
static mstate
get_free_list (void)
{
mstate result = free_list;
if (result != NULL)
{
(void)mutex_lock(&list_lock);
result = free_list;
if (result != NULL)
free_list = result->next_free;
(void)mutex_unlock(&list_lock);
if (result != NULL)
{
(void)mutex_lock(&result->mutex);
tsd_setspecific(arena_key, (void *)result);
THREAD_STAT(++(result->stat_lock_loop));
}
}
return result;
}
static mstate
internal_function
arena_get2(mstate a_tsd, size_t size, mstate avoid_arena)
{
mstate a;
#ifdef PER_THREAD
static size_t narenas_limit;
a = get_free_list ();
if (a == NULL)
{
/* Nothing immediately available, so generate a new arena. */
if (narenas_limit == 0)
{
if (mp_.arena_max != 0)
narenas_limit = mp_.arena_max;
else if (narenas > mp_.arena_test)
{
int n = __get_nprocs ();
if (n >= 1)
narenas_limit = NARENAS_FROM_NCORES (n);
else
/* We have no information about the system. Assume two
cores. */
narenas_limit = NARENAS_FROM_NCORES (2);
}
}
repeat:;
size_t n = narenas;
/* NB: the following depends on the fact that (size_t)0 - 1 is a
very large number and that the underflow is OK. If arena_max
is set the value of arena_test is irrelevant. If arena_test
is set but narenas is not yet larger or equal to arena_test
narenas_limit is 0. There is no possibility for narenas to
be too big for the test to always fail since there is not
enough address space to create that many arenas. */
if (__builtin_expect (n <= narenas_limit - 1, 0))
{
if (catomic_compare_and_exchange_bool_acq (&narenas, n + 1, n))
goto repeat;
a = _int_new_arena (size);
if (__builtin_expect (a == NULL, 0))
catomic_decrement (&narenas);
}
else
a = reused_arena (avoid_arena);
}
#else
if(!a_tsd)
a = a_tsd = &main_arena;
else {
a = a_tsd->next;
if(!a) {
/* This can only happen while initializing the new arena. */
(void)mutex_lock(&main_arena.mutex);
THREAD_STAT(++(main_arena.stat_lock_wait));
return &main_arena;
}
}
/* Check the global, circularly linked list for available arenas. */
bool retried = false;
repeat:
do {
if(!mutex_trylock(&a->mutex)) {
if (retried)
(void)mutex_unlock(&list_lock);
THREAD_STAT(++(a->stat_lock_loop));
tsd_setspecific(arena_key, (void *)a);
return a;
}
a = a->next;
} while(a != a_tsd);
/* If not even the list_lock can be obtained, try again. This can
happen during `atfork', or for example on systems where thread
creation makes it temporarily impossible to obtain _any_
locks. */
if(!retried && mutex_trylock(&list_lock)) {
/* We will block to not run in a busy loop. */
(void)mutex_lock(&list_lock);
/* Since we blocked there might be an arena available now. */
retried = true;
a = a_tsd;
goto repeat;
}
/* Nothing immediately available, so generate a new arena. */
a = _int_new_arena(size);
(void)mutex_unlock(&list_lock);
#endif
return a;
}
internal_function
new_heap(size_t size, size_t top_pad)
{
size_t page_mask = GLRO(dl_pagesize) - 1;
char *p1, *p2;
unsigned long ul;
heap_info *h;
if(size+top_pad < HEAP_MIN_SIZE)
size = HEAP_MIN_SIZE;
else if(size+top_pad <= HEAP_MAX_SIZE)
size += top_pad;
else if(size > HEAP_MAX_SIZE)
return 0;
else
size = HEAP_MAX_SIZE;
size = (size + page_mask) & ~page_mask;
/* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed.
No swap space needs to be reserved for the following large
mapping (on Linux, this is the case for all non-writable mappings
anyway). */
p2 = MAP_FAILED;
if(aligned_heap_area) {
p2 = (char *)MMAP(aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
MAP_NORESERVE);
aligned_heap_area = NULL;
if (p2 != MAP_FAILED && ((unsigned long)p2 & (HEAP_MAX_SIZE-1))) {
__munmap(p2, HEAP_MAX_SIZE);
p2 = MAP_FAILED;
}
}
if(p2 == MAP_FAILED) {
p1 = (char *)MMAP(0, HEAP_MAX_SIZE<<1, PROT_NONE, MAP_NORESERVE);
if(p1 != MAP_FAILED) {
p2 = (char *)(((unsigned long)p1 + (HEAP_MAX_SIZE-1))
& ~(HEAP_MAX_SIZE-1));
ul = p2 - p1;
if (ul)
__munmap(p1, ul);
else
aligned_heap_area = p2 + HEAP_MAX_SIZE;
__munmap(p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
} else {
/* Try to take the chance that an allocation of only HEAP_MAX_SIZE
is already aligned. */
p2 = (char *)MMAP(0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
if(p2 == MAP_FAILED)
return 0;
if((unsigned long)p2 & (HEAP_MAX_SIZE-1)) {
__munmap(p2, HEAP_MAX_SIZE);
return 0;
}
}
}
if(__mprotect(p2, size, PROT_READ|PROT_WRITE) != 0) {
__munmap(p2, HEAP_MAX_SIZE);
return 0;
}
h = (heap_info *)p2;
h->size = size;
h->mprotect_size = size;
THREAD_STAT(stat_n_heaps++);
return h;
}
static struct malloc_arena*
arena_get2(struct malloc_arena* a_tsd, size_t size)
{
struct malloc_arena* a;
int err;
if(!a_tsd)
a = a_tsd = &main_arena;
else {
a = a_tsd->next;
if(!a) {
/* This can only happen while initializing the new arena. */
(void)mutex_lock(&main_arena.mutex);
THREAD_STAT(++(main_arena.stat_lock_wait));
return &main_arena;
}
}
/* Check the global, circularly linked list for available arenas. */
repeat:
do {
if(!mutex_trylock(&a->mutex)) {
THREAD_STAT(++(a->stat_lock_loop));
tsd_setspecific(arena_key, (void *)a);
return a;
}
a = a->next;
} while(a != a_tsd);
/* If not even the list_lock can be obtained, try again. This can
happen during `atfork', or for example on systems where thread
creation makes it temporarily impossible to obtain _any_
locks. */
if(mutex_trylock(&list_lock)) {
a = a_tsd;
goto repeat;
}
(void)mutex_unlock(&list_lock);
/* Nothing immediately available, so generate a new arena. */
a = _int_new_arena(size);
if(!a)
return 0;
tsd_setspecific(arena_key, (void *)a);
mutex_init(&a->mutex);
err = mutex_lock(&a->mutex); /* remember result */
/* Add the new arena to the global list. */
(void)mutex_lock(&list_lock);
a->next = main_arena.next;
atomic_write_barrier ();
main_arena.next = a;
(void)mutex_unlock(&list_lock);
if(err) /* locking failed; keep arena for further attempts later */
return 0;
THREAD_STAT(++(a->stat_lock_loop));
return a;
}