bool market::lower_arena_priority ( arena& a, intptr_t new_priority, intptr_t old_priority ) {
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
if ( a.my_top_priority != old_priority ) {
assert_market_valid();
return false;
}
__TBB_ASSERT( a.my_top_priority > new_priority, NULL );
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
intptr_t p = a.my_top_priority;
update_arena_top_priority( a, new_priority );
if ( a.my_num_workers_requested > 0 ) {
if ( my_global_bottom_priority > new_priority ) {
my_global_bottom_priority = new_priority;
}
if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) {
// Global top level became empty
for ( --p; !my_priority_levels[p].workers_requested; --p ) continue;
__TBB_ASSERT( p >= my_global_bottom_priority, NULL );
update_global_top_priority(p);
}
update_allotment( p );
}
assert_market_valid();
return true;
}
bool market::lower_arena_priority ( arena& a, intptr_t new_priority, uintptr_t old_reload_epoch ) {
// TODO: replace the lock with a try_lock loop which performs a double check of the epoch
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
if ( a.my_reload_epoch != old_reload_epoch ) {
assert_market_valid();
return false;
}
__TBB_ASSERT( a.my_top_priority > new_priority, NULL );
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
intptr_t p = a.my_top_priority;
update_arena_top_priority( a, new_priority );
if ( a.my_num_workers_requested > 0 ) {
if ( my_global_bottom_priority > new_priority ) {
my_global_bottom_priority = new_priority;
}
if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) {
// Global top level became empty
for ( --p; !my_priority_levels[p].workers_requested; --p ) continue;
__TBB_ASSERT( p >= my_global_bottom_priority, NULL );
update_global_top_priority(p);
}
update_allotment( p );
}
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
assert_market_valid();
return true;
}
bool market::update_arena_priority ( arena& a, intptr_t new_priority ) {
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority || a.my_num_workers_requested <= 0, NULL );
assert_market_valid();
if ( a.my_top_priority == new_priority ) {
return false;
}
else if ( a.my_top_priority > new_priority ) {
if ( a.my_bottom_priority > new_priority )
a.my_bottom_priority = new_priority;
return false;
}
else if ( a.my_num_workers_requested <= 0 ) {
return false;
}
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
intptr_t p = a.my_top_priority;
intptr_t highest_affected_level = max(p, new_priority);
update_arena_top_priority( a, new_priority );
if ( my_global_top_priority < new_priority ) {
update_global_top_priority(new_priority);
}
else if ( my_global_top_priority == new_priority ) {
advance_global_reload_epoch();
}
else {
__TBB_ASSERT( new_priority < my_global_top_priority, NULL );
__TBB_ASSERT( new_priority > my_global_bottom_priority, NULL );
if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) {
// Global top level became empty
__TBB_ASSERT( my_global_bottom_priority < p, NULL );
for ( --p; !my_priority_levels[p].workers_requested; --p ) continue;
__TBB_ASSERT( p >= new_priority, NULL );
update_global_top_priority(p);
highest_affected_level = p;
}
}
if ( p == my_global_bottom_priority ) {
// Arena priority was increased from the global bottom level.
__TBB_ASSERT( p < new_priority, NULL ); // n
__TBB_ASSERT( new_priority <= my_global_top_priority, NULL );
while ( !my_priority_levels[my_global_bottom_priority].workers_requested )
++my_global_bottom_priority;
__TBB_ASSERT( my_global_bottom_priority <= new_priority, NULL );
__TBB_ASSERT( my_priority_levels[my_global_bottom_priority].workers_requested > 0, NULL );
}
update_allotment( highest_affected_level );
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
assert_market_valid();
return true;
}
arena* market::arena_in_need (
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
arena* prev_arena
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
)
{
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
assert_market_valid();
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
if ( prev_arena ) {
priority_level_info &pl = my_priority_levels[prev_arena->my_top_priority];
--prev_arena->my_num_workers_present;
--pl.workers_present;
if ( !--prev_arena->my_references && !prev_arena->my_num_workers_requested ) {
detach_arena( *a );
lock.release();
a->free_arena();
lock.acquire();
}
}
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
int p = my_global_top_priority;
arena *a = NULL;
do {
priority_level_info &pl = my_priority_levels[p];
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
__TBB_ASSERT( p >= my_lowest_populated_level, NULL );
if ( pl.workers_present >= pl.workers_requested )
continue;
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
a = arena_in_need( pl.arenas, pl.next_arena );
} while ( !a && --p >= my_global_bottom_priority );
return a;
}
void market::try_destroy_arena ( arena* a, uintptr_t aba_epoch ) {
__TBB_ASSERT ( a, NULL );
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
assert_market_valid();
#if __TBB_TASK_PRIORITY
for ( int p = my_global_top_priority; p >= my_global_bottom_priority; --p ) {
priority_level_info &pl = my_priority_levels[p];
arena_list_type &my_arenas = pl.arenas;
#endif /* __TBB_TASK_PRIORITY */
arena_list_type::iterator it = my_arenas.begin();
for ( ; it != my_arenas.end(); ++it ) {
if ( a == &*it ) {
if ( it->my_aba_epoch == aba_epoch ) {
// Arena is alive
if ( !a->my_num_workers_requested && !a->my_references ) {
__TBB_ASSERT( !a->my_num_workers_allotted && (a->my_pool_state == arena::SNAPSHOT_EMPTY || !a->my_max_num_workers), "Inconsistent arena state" );
// Arena is abandoned. Destroy it.
detach_arena( *a );
lock.release();
a->free_arena();
}
}
return;
}
}
#if __TBB_TASK_PRIORITY
}
#endif /* __TBB_TASK_PRIORITY */
}
void market::adjust_demand ( arena& a, int delta ) {
__TBB_ASSERT( theMarket, "market instance was destroyed prematurely?" );
if ( !delta )
return;
my_arenas_list_mutex.lock();
int prev_req = a.my_num_workers_requested;
a.my_num_workers_requested += delta;
if ( a.my_num_workers_requested <= 0 ) {
a.my_num_workers_allotted = 0;
if ( prev_req <= 0 ) {
my_arenas_list_mutex.unlock();
return;
}
delta = -prev_req;
}
#if __TBB_TASK_ARENA
else if ( prev_req < 0 ) {
delta = a.my_num_workers_requested;
}
#else /* __TBB_TASK_ARENA */
__TBB_ASSERT( prev_req >= 0, "Part-size request to RML?" );
#endif /* __TBB_TASK_ARENA */
#if __TBB_TASK_PRIORITY
intptr_t p = a.my_top_priority;
priority_level_info &pl = my_priority_levels[p];
pl.workers_requested += delta;
__TBB_ASSERT( pl.workers_requested >= 0, NULL );
#if !__TBB_TASK_ARENA
__TBB_ASSERT( a.my_num_workers_requested >= 0, NULL );
#else
//TODO: understand the assertion and modify
#endif
if ( a.my_num_workers_requested <= 0 ) {
if ( a.my_top_priority != normalized_normal_priority ) {
GATHER_STATISTIC( ++governor::local_scheduler_if_initialized()->my_counters.arena_prio_resets );
update_arena_top_priority( a, normalized_normal_priority );
}
a.my_bottom_priority = normalized_normal_priority;
}
if ( p == my_global_top_priority ) {
if ( !pl.workers_requested ) {
while ( --p >= my_global_bottom_priority && !my_priority_levels[p].workers_requested )
continue;
if ( p < my_global_bottom_priority )
reset_global_priority();
else
update_global_top_priority(p);
}
update_allotment( my_global_top_priority );
}
else if ( p > my_global_top_priority ) {
#if !__TBB_TASK_ARENA
__TBB_ASSERT( pl.workers_requested > 0, NULL );
#else
//TODO: understand the assertion and modify
#endif
update_global_top_priority(p);
a.my_num_workers_allotted = min( (int)my_max_num_workers, a.my_num_workers_requested );
my_priority_levels[p - 1].workers_available = my_max_num_workers - a.my_num_workers_allotted;
update_allotment( p - 1 );
}
else if ( p == my_global_bottom_priority ) {
if ( !pl.workers_requested ) {
while ( ++p <= my_global_top_priority && !my_priority_levels[p].workers_requested )
continue;
if ( p > my_global_top_priority )
reset_global_priority();
else {
my_global_bottom_priority = p;
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
my_lowest_populated_level = max( my_lowest_populated_level, p );
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
}
}
else
update_allotment( p );
}
else if ( p < my_global_bottom_priority ) {
__TBB_ASSERT( a.my_num_workers_requested > 0, NULL );
int prev_bottom = my_global_bottom_priority;
my_global_bottom_priority = p;
update_allotment( prev_bottom );
}
else {
__TBB_ASSERT( my_global_bottom_priority < p && p < my_global_top_priority, NULL );
update_allotment( p );
}
assert_market_valid();
#else /* !__TBB_TASK_PRIORITY */
my_total_demand += delta;
update_allotment();
#endif /* !__TBB_TASK_PRIORITY */
my_arenas_list_mutex.unlock();
// Must be called outside of any locks
my_server->adjust_job_count_estimate( delta );
GATHER_STATISTIC( governor::local_scheduler_if_initialized() ? ++governor::local_scheduler_if_initialized()->my_counters.gate_switches : 0 );
}