//------------------------------------------------------------------------
// micro_queue
//------------------------------------------------------------------------
void micro_queue::push( const void* item, ticket k, concurrent_queue_base& base ) {
k &= -concurrent_queue_rep::n_queue;
page* p = NULL;
size_t index = (k/concurrent_queue_rep::n_queue & base.items_per_page-1);
if ( !index ) {
size_t n = sizeof(page) + base.items_per_page*base.item_size;
p = static_cast<page*>(operator new( n ));
p->mask = 0;
p->next = NULL;
}
{
push_finalizer finalizer( *this, k+concurrent_queue_rep::n_queue );
spin_wait_until_eq( tail_counter, k );
if ( p ) {
spin_mutex::scoped_lock lock( page_mutex );
if ( page* q = tail_page )
q->next = p;
else
head_page = p;
tail_page = p;
} else {
p = tail_page;
}
base.copy_item( *p, index, item );
// If no exception was thrown, mark item as present.
p->mask |= uintptr(1)<<index;
}
}
task_group_context::~task_group_context () {
if ( my_kind != isolated ) {
generic_scheduler *s = (generic_scheduler*)my_owner;
if ( governor::is_set(s) ) {
// Local update of the context list
uintptr_t local_count_snapshot = s->local_cancel_count;
s->local_ctx_list_update = 1;
__TBB_full_memory_fence();
if ( s->nonlocal_ctx_list_update ) {
spin_mutex::scoped_lock lock(s->context_list_mutex);
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
s->local_ctx_list_update = 0;
}
else {
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
__TBB_store_with_release( s->local_ctx_list_update, 0 );
if ( local_count_snapshot != global_cancel_count ) {
// Another thread was propagating cancellation request when we removed
// ourselves from the list. We must ensure that it is not accessing us
// when this destructor finishes. We'll be able to acquire the lock
// below only after the other thread finishes with us.
spin_mutex::scoped_lock lock(s->context_list_mutex);
}
}
}
else {
// Nonlocal update of the context list
if ( __TBB_FetchAndStoreW(&my_kind, dying) == detached ) {
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
}
else {
__TBB_FetchAndAddW(&s->nonlocal_ctx_list_update, 1);
spin_wait_until_eq( s->local_ctx_list_update, 0u );
s->context_list_mutex.lock();
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
s->context_list_mutex.unlock();
__TBB_FetchAndAddW(&s->nonlocal_ctx_list_update, -1);
}
}
}
#if TBB_USE_DEBUG
my_version_and_traits = 0xDeadBeef;
#endif /* TBB_USE_DEBUG */
if ( my_exception )
my_exception->destroy();
if (itt_caller != ITT_CALLER_NULL) ITT_STACK(caller_destroy, itt_caller);
}
void WINAPI destroy_condvar_using_event( condition_variable_using_event* cv_event )
{
HANDLE my_event = cv_event->event;
EnterCriticalSection( &cv_event->mutex );
// NULL is an invalid HANDLE value
cv_event->event = NULL;
if( cv_event->n_waiters>0 ) {
LeaveCriticalSection( &cv_event->mutex );
spin_wait_until_eq( cv_event->n_waiters, 0 );
// make sure the last thread completes its access to cv
EnterCriticalSection( &cv_event->mutex );
}
LeaveCriticalSection( &cv_event->mutex );
CloseHandle( my_event );
}
bool micro_queue::pop( void* dst, ticket k, concurrent_queue_base& base ) {
k &= -concurrent_queue_rep::n_queue;
spin_wait_until_eq( head_counter, k );
spin_wait_while_eq( tail_counter, k );
page& p = *head_page;
__TBB_ASSERT( &p, NULL );
size_t index = (k/concurrent_queue_rep::n_queue & base.items_per_page-1);
bool success = false;
{
pop_finalizer finalizer( *this, k+concurrent_queue_rep::n_queue, index==base.items_per_page-1 ? &p : NULL );
if ( p.mask & uintptr(1)<<index ) {
success = true;
base.assign_and_destroy_item( dst, p, index );
}
}
return success;
}
task_group_context::~task_group_context () {
if ( __TBB_load_relaxed(my_kind) == binding_completed ) {
if ( governor::is_set(my_owner) ) {
// Local update of the context list
uintptr_t local_count_snapshot = my_owner->my_context_state_propagation_epoch;
my_owner->my_local_ctx_list_update.store<relaxed>(1);
// Prevent load of nonlocal update flag from being hoisted before the
// store to local update flag.
atomic_fence();
if ( my_owner->my_nonlocal_ctx_list_update.load<relaxed>() ) {
spin_mutex::scoped_lock lock(my_owner->my_context_list_mutex);
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
my_owner->my_local_ctx_list_update.store<relaxed>(0);
}
else {
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
// Release fence is necessary so that update of our neighbors in
// the context list was committed when possible concurrent destroyer
// proceeds after local update flag is reset by the following store.
my_owner->my_local_ctx_list_update.store<release>(0);
if ( local_count_snapshot != the_context_state_propagation_epoch ) {
// Another thread was propagating cancellation request when we removed
// ourselves from the list. We must ensure that it is not accessing us
// when this destructor finishes. We'll be able to acquire the lock
// below only after the other thread finishes with us.
spin_mutex::scoped_lock lock(my_owner->my_context_list_mutex);
}
}
}
else {
// Nonlocal update of the context list
// Synchronizes with generic_scheduler::cleanup_local_context_list()
// TODO: evaluate and perhaps relax, or add some lock instead
if ( internal::as_atomic(my_kind).fetch_and_store(dying) == detached ) {
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
}
else {
//TODO: evaluate and perhaps relax
my_owner->my_nonlocal_ctx_list_update.fetch_and_increment<full_fence>();
//TODO: evaluate and perhaps remove
spin_wait_until_eq( my_owner->my_local_ctx_list_update, 0u );
my_owner->my_context_list_mutex.lock();
my_node.my_prev->my_next = my_node.my_next;
my_node.my_next->my_prev = my_node.my_prev;
my_owner->my_context_list_mutex.unlock();
//TODO: evaluate and perhaps relax
my_owner->my_nonlocal_ctx_list_update.fetch_and_decrement<full_fence>();
}
}
}
#if __TBB_FP_CONTEXT
internal::punned_cast<cpu_ctl_env*>(&my_cpu_ctl_env)->~cpu_ctl_env();
#endif
poison_value(my_version_and_traits);
if ( my_exception )
my_exception->destroy();
ITT_STACK(itt_caller != ITT_CALLER_NULL, caller_destroy, itt_caller);
}
