arena* market::arena_in_need (
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
arena* prev_arena
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
)
{
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
assert_market_valid();
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
if ( prev_arena ) {
priority_level_info &pl = my_priority_levels[prev_arena->my_top_priority];
--prev_arena->my_num_workers_present;
--pl.workers_present;
if ( !--prev_arena->my_references && !prev_arena->my_num_workers_requested ) {
detach_arena( *a );
lock.release();
a->free_arena();
lock.acquire();
}
}
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
int p = my_global_top_priority;
arena *a = NULL;
do {
priority_level_info &pl = my_priority_levels[p];
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
__TBB_ASSERT( p >= my_lowest_populated_level, NULL );
if ( pl.workers_present >= pl.workers_requested )
continue;
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
a = arena_in_need( pl.arenas, pl.next_arena );
} while ( !a && --p >= my_global_bottom_priority );
return a;
}
void market::try_destroy_arena ( arena* a, uintptr_t aba_epoch ) {
__TBB_ASSERT ( a, NULL );
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
assert_market_valid();
#if __TBB_TASK_PRIORITY
for ( int p = my_global_top_priority; p >= my_global_bottom_priority; --p ) {
priority_level_info &pl = my_priority_levels[p];
arena_list_type &my_arenas = pl.arenas;
#endif /* __TBB_TASK_PRIORITY */
arena_list_type::iterator it = my_arenas.begin();
for ( ; it != my_arenas.end(); ++it ) {
if ( a == &*it ) {
if ( it->my_aba_epoch == aba_epoch ) {
// Arena is alive
if ( !a->my_num_workers_requested && !a->my_references ) {
__TBB_ASSERT( !a->my_num_workers_allotted && (a->my_pool_state == arena::SNAPSHOT_EMPTY || !a->my_max_num_workers), "Inconsistent arena state" );
// Arena is abandoned. Destroy it.
detach_arena( *a );
lock.release();
a->free_arena();
}
}
return;
}
}
#if __TBB_TASK_PRIORITY
}
#endif /* __TBB_TASK_PRIORITY */
}
/* Remove an arena from free_list. */
static mstate
get_free_list (void)
{
mstate replaced_arena = thread_arena;
mstate result = free_list;
if (result != NULL)
{
__libc_lock_lock (free_list_lock);
result = free_list;
if (result != NULL)
{
free_list = result->next_free;
/* The arena will be attached to this thread. */
assert (result->attached_threads == 0);
result->attached_threads = 1;
detach_arena (replaced_arena);
}
__libc_lock_unlock (free_list_lock);
if (result != NULL)
{
LIBC_PROBE (memory_arena_reuse_free_list, 1, result);
__libc_lock_lock (result->mutex);
thread_arena = result;
}
}
return result;
}
/* Lock and return an arena that can be reused for memory allocation.
Avoid AVOID_ARENA as we have already failed to allocate memory in
it and it is currently locked. */
static mstate
reused_arena (mstate avoid_arena)
{
mstate result;
/* FIXME: Access to next_to_use suffers from data races. */
static mstate next_to_use;
if (next_to_use == NULL)
next_to_use = &main_arena;
/* Iterate over all arenas (including those linked from
free_list). */
result = next_to_use;
do
{
if (!__libc_lock_trylock (result->mutex))
goto out;
/* FIXME: This is a data race, see _int_new_arena. */
result = result->next;
}
while (result != next_to_use);
/* Avoid AVOID_ARENA as we have already failed to allocate memory
in that arena and it is currently locked. */
if (result == avoid_arena)
result = result->next;
/* No arena available without contention. Wait for the next in line. */
LIBC_PROBE (memory_arena_reuse_wait, 3, &result->mutex, result, avoid_arena);
__libc_lock_lock (result->mutex);
out:
/* Attach the arena to the current thread. */
{
/* Update the arena thread attachment counters. */
mstate replaced_arena = thread_arena;
__libc_lock_lock (free_list_lock);
detach_arena (replaced_arena);
/* We may have picked up an arena on the free list. We need to
preserve the invariant that no arena on the free list has a
positive attached_threads counter (otherwise,
arena_thread_freeres cannot use the counter to determine if the
arena needs to be put on the free list). We unconditionally
remove the selected arena from the free list. The caller of
reused_arena checked the free list and observed it to be empty,
so the list is very short. */
remove_from_free_list (result);
++result->attached_threads;
__libc_lock_unlock (free_list_lock);
}
LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena);
thread_arena = result;
next_to_use = result->next;
return result;
}
static mstate
_int_new_arena (size_t size)
{
mstate a;
heap_info *h;
char *ptr;
unsigned long misalign;
h = new_heap (size + (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT),
mp_.top_pad);
if (!h)
{
/* Maybe size is too large to fit in a single heap. So, just try
to create a minimally-sized arena and let _int_malloc() attempt
to deal with the large request via mmap_chunk(). */
h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad);
if (!h)
return 0;
}
a = h->ar_ptr = (mstate) (h + 1);
malloc_init_state (a);
a->attached_threads = 1;
/*a->next = NULL;*/
a->system_mem = a->max_system_mem = h->size;
/* Set up the top chunk, with proper alignment. */
ptr = (char *) (a + 1);
misalign = (unsigned long) chunk2mem (ptr) & MALLOC_ALIGN_MASK;
if (misalign > 0)
ptr += MALLOC_ALIGNMENT - misalign;
top (a) = (mchunkptr) ptr;
set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE);
LIBC_PROBE (memory_arena_new, 2, a, size);
mstate replaced_arena = thread_arena;
thread_arena = a;
__libc_lock_init (a->mutex);
__libc_lock_lock (list_lock);
/* Add the new arena to the global list. */
a->next = main_arena.next;
/* FIXME: The barrier is an attempt to synchronize with read access
in reused_arena, which does not acquire list_lock while
traversing the list. */
atomic_write_barrier ();
main_arena.next = a;
__libc_lock_unlock (list_lock);
__libc_lock_lock (free_list_lock);
detach_arena (replaced_arena);
__libc_lock_unlock (free_list_lock);
/* Lock this arena. NB: Another thread may have been attached to
this arena because the arena is now accessible from the
main_arena.next list and could have been picked by reused_arena.
This can only happen for the last arena created (before the arena
limit is reached). At this point, some arena has to be attached
to two threads. We could acquire the arena lock before list_lock
to make it less likely that reused_arena picks this new arena,
but this could result in a deadlock with
__malloc_fork_lock_parent. */
__libc_lock_lock (a->mutex);
return a;
}
