bool market::update_arena_priority ( arena& a, intptr_t new_priority ) { arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex); __TBB_ASSERT( my_global_top_priority >= a.my_top_priority || a.my_num_workers_requested <= 0, NULL ); assert_market_valid(); if ( a.my_top_priority == new_priority ) { return false; } else if ( a.my_top_priority > new_priority ) { if ( a.my_bottom_priority > new_priority ) a.my_bottom_priority = new_priority; return false; } else if ( a.my_num_workers_requested <= 0 ) { return false; } __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); intptr_t p = a.my_top_priority; intptr_t highest_affected_level = max(p, new_priority); update_arena_top_priority( a, new_priority ); if ( my_global_top_priority < new_priority ) { update_global_top_priority(new_priority); } else if ( my_global_top_priority == new_priority ) { advance_global_reload_epoch(); } else { __TBB_ASSERT( new_priority < my_global_top_priority, NULL ); __TBB_ASSERT( new_priority > my_global_bottom_priority, NULL ); if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) { // Global top level became empty __TBB_ASSERT( my_global_bottom_priority < p, NULL ); for ( --p; !my_priority_levels[p].workers_requested; --p ) continue; __TBB_ASSERT( p >= new_priority, NULL ); update_global_top_priority(p); highest_affected_level = p; } } if ( p == my_global_bottom_priority ) { // Arena priority was increased from the global bottom level. __TBB_ASSERT( p < new_priority, NULL ); // n __TBB_ASSERT( new_priority <= my_global_top_priority, NULL ); while ( !my_priority_levels[my_global_bottom_priority].workers_requested ) ++my_global_bottom_priority; __TBB_ASSERT( my_global_bottom_priority <= new_priority, NULL ); __TBB_ASSERT( my_priority_levels[my_global_bottom_priority].workers_requested > 0, NULL ); } update_allotment( highest_affected_level ); __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); assert_market_valid(); return true; }
bool market::lower_arena_priority ( arena& a, intptr_t new_priority, intptr_t old_priority ) { arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex); if ( a.my_top_priority != old_priority ) { assert_market_valid(); return false; } __TBB_ASSERT( a.my_top_priority > new_priority, NULL ); __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); intptr_t p = a.my_top_priority; update_arena_top_priority( a, new_priority ); if ( a.my_num_workers_requested > 0 ) { if ( my_global_bottom_priority > new_priority ) { my_global_bottom_priority = new_priority; } if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) { // Global top level became empty for ( --p; !my_priority_levels[p].workers_requested; --p ) continue; __TBB_ASSERT( p >= my_global_bottom_priority, NULL ); update_global_top_priority(p); } update_allotment( p ); } assert_market_valid(); return true; }
inline void market::reset_global_priority () { my_global_bottom_priority = normalized_normal_priority; update_global_top_priority(normalized_normal_priority); #if __TBB_TRACK_PRIORITY_LEVEL_SATURATION my_lowest_populated_level = normalized_normal_priority; #endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */ }
bool market::lower_arena_priority ( arena& a, intptr_t new_priority, uintptr_t old_reload_epoch ) { // TODO: replace the lock with a try_lock loop which performs a double check of the epoch arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex); if ( a.my_reload_epoch != old_reload_epoch ) { assert_market_valid(); return false; } __TBB_ASSERT( a.my_top_priority > new_priority, NULL ); __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); intptr_t p = a.my_top_priority; update_arena_top_priority( a, new_priority ); if ( a.my_num_workers_requested > 0 ) { if ( my_global_bottom_priority > new_priority ) { my_global_bottom_priority = new_priority; } if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) { // Global top level became empty for ( --p; !my_priority_levels[p].workers_requested; --p ) continue; __TBB_ASSERT( p >= my_global_bottom_priority, NULL ); update_global_top_priority(p); } update_allotment( p ); } __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); assert_market_valid(); return true; }
void market::adjust_demand ( arena& a, int delta ) { __TBB_ASSERT( theMarket, "market instance was destroyed prematurely?" ); if ( !delta ) return; my_arenas_list_mutex.lock(); int prev_req = a.my_num_workers_requested; a.my_num_workers_requested += delta; if ( a.my_num_workers_requested <= 0 ) { a.my_num_workers_allotted = 0; if ( prev_req <= 0 ) { my_arenas_list_mutex.unlock(); return; } delta = -prev_req; } #if __TBB_TASK_ARENA else if ( prev_req < 0 ) { delta = a.my_num_workers_requested; } #else /* __TBB_TASK_ARENA */ __TBB_ASSERT( prev_req >= 0, "Part-size request to RML?" ); #endif /* __TBB_TASK_ARENA */ #if __TBB_TASK_PRIORITY intptr_t p = a.my_top_priority; priority_level_info &pl = my_priority_levels[p]; pl.workers_requested += delta; __TBB_ASSERT( pl.workers_requested >= 0, NULL ); #if !__TBB_TASK_ARENA __TBB_ASSERT( a.my_num_workers_requested >= 0, NULL ); #else //TODO: understand the assertion and modify #endif if ( a.my_num_workers_requested <= 0 ) { if ( a.my_top_priority != normalized_normal_priority ) { GATHER_STATISTIC( ++governor::local_scheduler_if_initialized()->my_counters.arena_prio_resets ); update_arena_top_priority( a, normalized_normal_priority ); } a.my_bottom_priority = normalized_normal_priority; } if ( p == my_global_top_priority ) { if ( !pl.workers_requested ) { while ( --p >= my_global_bottom_priority && !my_priority_levels[p].workers_requested ) continue; if ( p < my_global_bottom_priority ) reset_global_priority(); else update_global_top_priority(p); } update_allotment( my_global_top_priority ); } else if ( p > my_global_top_priority ) { #if !__TBB_TASK_ARENA __TBB_ASSERT( pl.workers_requested > 0, NULL ); #else //TODO: understand the assertion and modify #endif update_global_top_priority(p); a.my_num_workers_allotted = min( (int)my_max_num_workers, a.my_num_workers_requested ); my_priority_levels[p - 1].workers_available = my_max_num_workers - a.my_num_workers_allotted; update_allotment( p - 1 ); } else if ( p == my_global_bottom_priority ) { if ( !pl.workers_requested ) { while ( ++p <= my_global_top_priority && !my_priority_levels[p].workers_requested ) continue; if ( p > my_global_top_priority ) reset_global_priority(); else { my_global_bottom_priority = p; #if __TBB_TRACK_PRIORITY_LEVEL_SATURATION my_lowest_populated_level = max( my_lowest_populated_level, p ); #endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */ } } else update_allotment( p ); } else if ( p < my_global_bottom_priority ) { __TBB_ASSERT( a.my_num_workers_requested > 0, NULL ); int prev_bottom = my_global_bottom_priority; my_global_bottom_priority = p; update_allotment( prev_bottom ); } else { __TBB_ASSERT( my_global_bottom_priority < p && p < my_global_top_priority, NULL ); update_allotment( p ); } assert_market_valid(); #else /* !__TBB_TASK_PRIORITY */ my_total_demand += delta; update_allotment(); #endif /* !__TBB_TASK_PRIORITY */ my_arenas_list_mutex.unlock(); // Must be called outside of any locks my_server->adjust_job_count_estimate( delta ); GATHER_STATISTIC( governor::local_scheduler_if_initialized() ? ++governor::local_scheduler_if_initialized()->my_counters.gate_switches : 0 ); }