//! Release speculative mutex
void x86_rtm_rw_mutex::internal_release(x86_rtm_rw_mutex::scoped_lock& s) {
switch(s.transaction_state) {
case RTM_transacting_writer:
case RTM_transacting_reader:
{
__TBB_ASSERT(__TBB_machine_is_in_transaction(), "transaction_state && not speculating");
#if __TBB_RW_MUTEX_DELAY_TEST
if(s.transaction_state == RTM_transacting_reader) {
if(this->w_flag) __TBB_machine_transaction_conflict_abort();
} else {
if(this->state) __TBB_machine_transaction_conflict_abort();
}
#endif
__TBB_machine_end_transaction();
s.my_scoped_lock.internal_set_mutex(NULL);
}
break;
case RTM_real_reader:
__TBB_ASSERT(!this->w_flag, "w_flag set but read lock acquired");
s.my_scoped_lock.release();
break;
case RTM_real_writer:
__TBB_ASSERT(this->w_flag, "w_flag unset but write lock acquired");
this->w_flag = false;
s.my_scoped_lock.release();
break;
case RTM_not_in_mutex:
__TBB_ASSERT(false, "RTM_not_in_mutex, but in release");
default:
__TBB_ASSERT(false, "invalid transaction_state");
}
s.transaction_state = RTM_not_in_mutex;
}
//! Acquire read lock on given mutex.
// only_speculate : true if we are doing a try_acquire. If true and we fail to speculate, don't
// really acquire the lock, return and do a try_acquire on the contained spin_rw_mutex. If
// the lock is already held by a writer, just return.
void x86_rtm_rw_mutex::internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate) {
__TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction");
if(tbb::internal::governor::speculation_enabled()) {
int num_retries = 0;
unsigned int abort_code;
do {
tbb::internal::atomic_backoff backoff;
// if in try_acquire, and lock is held as writer, don't attempt to speculate.
if(w_flag) {
if(only_speculate) return;
do {
backoff.pause(); // test the spin_rw_mutex (real readers or writers)
} while(w_flag);
}
// _xbegin returns -1 on success or the abort code, so capture it
if((abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) )
{
// started speculation
#if !__TBB_RW_MUTEX_DELAY_TEST
if(w_flag) { // add w_flag to read-set.
__TBB_machine_transaction_conflict_abort(); // writer grabbed the lock, so abort.
}
#endif
s.transaction_state = RTM_transacting_reader;
// Don not wrap the following assignment to a function,
// because it can abort the transaction in debug. Need mutex for release().
s.my_scoped_lock.mutex = this;
return; // successfully started speculation
}
// fallback path
// retry only if there is any hope of getting into a transaction soon
// Retry in the following cases (from Section 8.3.5 of Intel(R)
// Architecture Instruction Set Extensions Programming Reference):
// 1. abort caused by XABORT instruction (bit 0 of EAX register is set)
// 2. the transaction may succeed on a retry (bit 1 of EAX register is set)
// 3. if another logical processor conflicted with a memory address
// that was part of the transaction that aborted (bit 2 of EAX register is set)
// That is, retry if (abort_code & 0x7) is non-zero
++num_retries;
} while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_read) );
}
if(only_speculate) return;
s.my_scoped_lock.acquire( *this, false );
s.transaction_state = RTM_real_reader;
}
//! Downgrade writer to a reader.
bool x86_rtm_rw_mutex::internal_downgrade(x86_rtm_rw_mutex::scoped_lock& s) {
switch(s.transaction_state) {
case RTM_real_writer:
s.transaction_state = RTM_real_reader;
__TBB_ASSERT(w_flag, "Before downgrade_to_reader w_flag not true");
w_flag = false;
return s.my_scoped_lock.downgrade_to_reader();
case RTM_transacting_writer:
#if __TBB_RW_MUTEX_DELAY_TEST
if(this->state) { // a reader or writer has acquired mutex for real.
__TBB_machine_transaction_conflict_abort();
}
#endif
s.transaction_state = RTM_transacting_reader;
return true;
default:
__TBB_ASSERT(false, "Invalid state for downgrade");
return false;
}
}
//! Acquire write lock on the given mutex.
void x86_rtm_rw_mutex::internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate)
{
__TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction");
if(tbb::internal::governor::speculation_enabled()) {
int num_retries = 0;
unsigned int abort_code;
do {
tbb::internal::atomic_backoff backoff;
if(this->state) {
if(only_speculate) return;
do {
backoff.pause(); // test the spin_rw_mutex (real readers or writers)
} while(this->state);
}
// _xbegin returns -1 on success or the abort code, so capture it
if(( abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) )
{
// started speculation
#if !__TBB_RW_MUTEX_DELAY_TEST
if(this->state) { // add spin_rw_mutex to read-set.
// reader or writer grabbed the lock, so abort.
__TBB_machine_transaction_conflict_abort();
}
#endif
s.transaction_state = RTM_transacting_writer;
// Don not wrap the following assignment to a function,
// because it can abort the transaction in debug. Need mutex for release().
s.my_scoped_lock.mutex = this;
return; // successfully started speculation
}
++num_retries;
} while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_write) );
}
if(only_speculate) return; // should apply a real try_lock...
s.my_scoped_lock.acquire(*this, true); // kill transactional writers
__TBB_ASSERT(!w_flag, "After acquire for write, w_flag already true");
w_flag = true; // kill transactional readers
s.transaction_state = RTM_real_writer;
return;
}
