bool market::lower_arena_priority ( arena& a, intptr_t new_priority, intptr_t old_priority ) {
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
if ( a.my_top_priority != old_priority ) {
assert_market_valid();
return false;
}
__TBB_ASSERT( a.my_top_priority > new_priority, NULL );
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
intptr_t p = a.my_top_priority;
update_arena_top_priority( a, new_priority );
if ( a.my_num_workers_requested > 0 ) {
if ( my_global_bottom_priority > new_priority ) {
my_global_bottom_priority = new_priority;
}
if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) {
// Global top level became empty
for ( --p; !my_priority_levels[p].workers_requested; --p ) continue;
__TBB_ASSERT( p >= my_global_bottom_priority, NULL );
update_global_top_priority(p);
}
update_allotment( p );
}
assert_market_valid();
return true;
}
void market::update_allotment ( intptr_t highest_affected_priority ) {
intptr_t i = highest_affected_priority;
int available = my_priority_levels[i].workers_available;
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
my_lowest_populated_level = my_global_bottom_priority;
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
for ( ; i >= my_global_bottom_priority; --i ) {
priority_level_info &pl = my_priority_levels[i];
pl.workers_available = available;
if ( pl.workers_requested ) {
update_allotment( pl.arenas, pl.workers_requested, available );
available -= pl.workers_requested;
if ( available < 0 ) {
available = 0;
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
my_lowest_populated_level = i;
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
break;
}
}
}
__TBB_ASSERT( i <= my_global_bottom_priority || !available, NULL );
for ( --i; i >= my_global_bottom_priority; --i ) {
priority_level_info &pl = my_priority_levels[i];
pl.workers_available = 0;
arena_list_type::iterator it = pl.arenas.begin();
for ( ; it != pl.arenas.end(); ++it ) {
__TBB_ASSERT( it->my_num_workers_requested || !it->my_num_workers_allotted, NULL );
it->my_num_workers_allotted = 0;
}
}
}
bool market::lower_arena_priority ( arena& a, intptr_t new_priority, uintptr_t old_reload_epoch ) {
// TODO: replace the lock with a try_lock loop which performs a double check of the epoch
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
if ( a.my_reload_epoch != old_reload_epoch ) {
assert_market_valid();
return false;
}
__TBB_ASSERT( a.my_top_priority > new_priority, NULL );
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
intptr_t p = a.my_top_priority;
update_arena_top_priority( a, new_priority );
if ( a.my_num_workers_requested > 0 ) {
if ( my_global_bottom_priority > new_priority ) {
my_global_bottom_priority = new_priority;
}
if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) {
// Global top level became empty
for ( --p; !my_priority_levels[p].workers_requested; --p ) continue;
__TBB_ASSERT( p >= my_global_bottom_priority, NULL );
update_global_top_priority(p);
}
update_allotment( p );
}
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
assert_market_valid();
return true;
}
/** The balancing algorithm may be liable to data races. However the aberrations
caused by the races are not fatal and generally only temporarily affect fairness
of the workers distribution among arenas. **/
void market::adjust_demand ( arena& a, int delta ) {
__TBB_ASSERT( theMarket, "market instance was destroyed prematurely?" );
a.my_num_workers_requested += delta;
my_total_demand += delta;
update_allotment( my_max_num_workers );
// Must be called outside of any locks
my_server->adjust_job_count_estimate( delta );
}
bool market::update_arena_priority ( arena& a, intptr_t new_priority ) {
arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex);
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority || a.my_num_workers_requested <= 0, NULL );
assert_market_valid();
if ( a.my_top_priority == new_priority ) {
return false;
}
else if ( a.my_top_priority > new_priority ) {
if ( a.my_bottom_priority > new_priority )
a.my_bottom_priority = new_priority;
return false;
}
else if ( a.my_num_workers_requested <= 0 ) {
return false;
}
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
intptr_t p = a.my_top_priority;
intptr_t highest_affected_level = max(p, new_priority);
update_arena_top_priority( a, new_priority );
if ( my_global_top_priority < new_priority ) {
update_global_top_priority(new_priority);
}
else if ( my_global_top_priority == new_priority ) {
advance_global_reload_epoch();
}
else {
__TBB_ASSERT( new_priority < my_global_top_priority, NULL );
__TBB_ASSERT( new_priority > my_global_bottom_priority, NULL );
if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) {
// Global top level became empty
__TBB_ASSERT( my_global_bottom_priority < p, NULL );
for ( --p; !my_priority_levels[p].workers_requested; --p ) continue;
__TBB_ASSERT( p >= new_priority, NULL );
update_global_top_priority(p);
highest_affected_level = p;
}
}
if ( p == my_global_bottom_priority ) {
// Arena priority was increased from the global bottom level.
__TBB_ASSERT( p < new_priority, NULL ); // n
__TBB_ASSERT( new_priority <= my_global_top_priority, NULL );
while ( !my_priority_levels[my_global_bottom_priority].workers_requested )
++my_global_bottom_priority;
__TBB_ASSERT( my_global_bottom_priority <= new_priority, NULL );
__TBB_ASSERT( my_priority_levels[my_global_bottom_priority].workers_requested > 0, NULL );
}
update_allotment( highest_affected_level );
__TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL );
assert_market_valid();
return true;
}
void market::adjust_demand ( arena& ab, int delta ) {
if (!aa) aa = &ab;
arena &a = *aa;
if (delta == -12345)
delta = 0 - a.my_num_workers_requested;
__TBB_ASSERT( theMarket, "market instance was destroyed prematurely?" );
//runtime_warning("** %p %d %d %d %d --> ", &a, (int)a.my_num_workers_requested, (int)my_total_demand, (int)a.my_num_workers_allotted, delta);
a.my_num_workers_requested += delta;
my_total_demand += delta;
update_allotment( my_max_num_workers );
//runtime_warning("%d %d %d", (int)a.my_num_workers_requested, (int)my_total_demand, (int)a.my_num_workers_allotted);
// Must be called outside of any locks
my_server->adjust_job_count_estimate( delta );
GATHER_STATISTIC( governor::local_scheduler_if_initialized() ? ++governor::local_scheduler_if_initialized()->my_counters.gate_switches : 0 );
}
void market::adjust_demand ( arena& a, int delta ) {
__TBB_ASSERT( theMarket, "market instance was destroyed prematurely?" );
if ( !delta )
return;
my_arenas_list_mutex.lock();
int prev_req = a.my_num_workers_requested;
a.my_num_workers_requested += delta;
if ( a.my_num_workers_requested <= 0 ) {
a.my_num_workers_allotted = 0;
if ( prev_req <= 0 ) {
my_arenas_list_mutex.unlock();
return;
}
delta = -prev_req;
}
#if __TBB_TASK_ARENA
else if ( prev_req < 0 ) {
delta = a.my_num_workers_requested;
}
#else /* __TBB_TASK_ARENA */
__TBB_ASSERT( prev_req >= 0, "Part-size request to RML?" );
#endif /* __TBB_TASK_ARENA */
#if __TBB_TASK_PRIORITY
intptr_t p = a.my_top_priority;
priority_level_info &pl = my_priority_levels[p];
pl.workers_requested += delta;
__TBB_ASSERT( pl.workers_requested >= 0, NULL );
#if !__TBB_TASK_ARENA
__TBB_ASSERT( a.my_num_workers_requested >= 0, NULL );
#else
//TODO: understand the assertion and modify
#endif
if ( a.my_num_workers_requested <= 0 ) {
if ( a.my_top_priority != normalized_normal_priority ) {
GATHER_STATISTIC( ++governor::local_scheduler_if_initialized()->my_counters.arena_prio_resets );
update_arena_top_priority( a, normalized_normal_priority );
}
a.my_bottom_priority = normalized_normal_priority;
}
if ( p == my_global_top_priority ) {
if ( !pl.workers_requested ) {
while ( --p >= my_global_bottom_priority && !my_priority_levels[p].workers_requested )
continue;
if ( p < my_global_bottom_priority )
reset_global_priority();
else
update_global_top_priority(p);
}
update_allotment( my_global_top_priority );
}
else if ( p > my_global_top_priority ) {
#if !__TBB_TASK_ARENA
__TBB_ASSERT( pl.workers_requested > 0, NULL );
#else
//TODO: understand the assertion and modify
#endif
update_global_top_priority(p);
a.my_num_workers_allotted = min( (int)my_max_num_workers, a.my_num_workers_requested );
my_priority_levels[p - 1].workers_available = my_max_num_workers - a.my_num_workers_allotted;
update_allotment( p - 1 );
}
else if ( p == my_global_bottom_priority ) {
if ( !pl.workers_requested ) {
while ( ++p <= my_global_top_priority && !my_priority_levels[p].workers_requested )
continue;
if ( p > my_global_top_priority )
reset_global_priority();
else {
my_global_bottom_priority = p;
#if __TBB_TRACK_PRIORITY_LEVEL_SATURATION
my_lowest_populated_level = max( my_lowest_populated_level, p );
#endif /* __TBB_TRACK_PRIORITY_LEVEL_SATURATION */
}
}
else
update_allotment( p );
}
else if ( p < my_global_bottom_priority ) {
__TBB_ASSERT( a.my_num_workers_requested > 0, NULL );
int prev_bottom = my_global_bottom_priority;
my_global_bottom_priority = p;
update_allotment( prev_bottom );
}
else {
__TBB_ASSERT( my_global_bottom_priority < p && p < my_global_top_priority, NULL );
update_allotment( p );
}
assert_market_valid();
#else /* !__TBB_TASK_PRIORITY */
my_total_demand += delta;
update_allotment();
#endif /* !__TBB_TASK_PRIORITY */
my_arenas_list_mutex.unlock();
// Must be called outside of any locks
my_server->adjust_job_count_estimate( delta );
GATHER_STATISTIC( governor::local_scheduler_if_initialized() ? ++governor::local_scheduler_if_initialized()->my_counters.gate_switches : 0 );
}