//! Release speculative mutex void x86_rtm_rw_mutex::internal_release(x86_rtm_rw_mutex::scoped_lock& s) { switch(s.transaction_state) { case RTM_transacting_writer: case RTM_transacting_reader: { __TBB_ASSERT(__TBB_machine_is_in_transaction(), "transaction_state && not speculating"); #if __TBB_RW_MUTEX_DELAY_TEST if(s.transaction_state == RTM_transacting_reader) { if(this->w_flag) __TBB_machine_transaction_conflict_abort(); } else { if(this->state) __TBB_machine_transaction_conflict_abort(); } #endif __TBB_machine_end_transaction(); s.my_scoped_lock.internal_set_mutex(NULL); } break; case RTM_real_reader: __TBB_ASSERT(!this->w_flag, "w_flag set but read lock acquired"); s.my_scoped_lock.release(); break; case RTM_real_writer: __TBB_ASSERT(this->w_flag, "w_flag unset but write lock acquired"); this->w_flag = false; s.my_scoped_lock.release(); break; case RTM_not_in_mutex: __TBB_ASSERT(false, "RTM_not_in_mutex, but in release"); default: __TBB_ASSERT(false, "invalid transaction_state"); } s.transaction_state = RTM_not_in_mutex; }
//! Acquire read lock on given mutex. // only_speculate : true if we are doing a try_acquire. If true and we fail to speculate, don't // really acquire the lock, return and do a try_acquire on the contained spin_rw_mutex. If // the lock is already held by a writer, just return. void x86_rtm_rw_mutex::internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate) { __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction"); if(tbb::internal::governor::speculation_enabled()) { int num_retries = 0; unsigned int abort_code; do { tbb::internal::atomic_backoff backoff; // if in try_acquire, and lock is held as writer, don't attempt to speculate. if(w_flag) { if(only_speculate) return; do { backoff.pause(); // test the spin_rw_mutex (real readers or writers) } while(w_flag); } // _xbegin returns -1 on success or the abort code, so capture it if((abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) ) { // started speculation #if !__TBB_RW_MUTEX_DELAY_TEST if(w_flag) { // add w_flag to read-set. __TBB_machine_transaction_conflict_abort(); // writer grabbed the lock, so abort. } #endif s.transaction_state = RTM_transacting_reader; // Don not wrap the following assignment to a function, // because it can abort the transaction in debug. Need mutex for release(). s.my_scoped_lock.mutex = this; return; // successfully started speculation } // fallback path // retry only if there is any hope of getting into a transaction soon // Retry in the following cases (from Section 8.3.5 of Intel(R) // Architecture Instruction Set Extensions Programming Reference): // 1. abort caused by XABORT instruction (bit 0 of EAX register is set) // 2. the transaction may succeed on a retry (bit 1 of EAX register is set) // 3. if another logical processor conflicted with a memory address // that was part of the transaction that aborted (bit 2 of EAX register is set) // That is, retry if (abort_code & 0x7) is non-zero ++num_retries; } while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_read) ); } if(only_speculate) return; s.my_scoped_lock.acquire( *this, false ); s.transaction_state = RTM_real_reader; }
//! Downgrade writer to a reader. bool x86_rtm_rw_mutex::internal_downgrade(x86_rtm_rw_mutex::scoped_lock& s) { switch(s.transaction_state) { case RTM_real_writer: s.transaction_state = RTM_real_reader; __TBB_ASSERT(w_flag, "Before downgrade_to_reader w_flag not true"); w_flag = false; return s.my_scoped_lock.downgrade_to_reader(); case RTM_transacting_writer: #if __TBB_RW_MUTEX_DELAY_TEST if(this->state) { // a reader or writer has acquired mutex for real. __TBB_machine_transaction_conflict_abort(); } #endif s.transaction_state = RTM_transacting_reader; return true; default: __TBB_ASSERT(false, "Invalid state for downgrade"); return false; } }
//! Acquire write lock on the given mutex. void x86_rtm_rw_mutex::internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate) { __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction"); if(tbb::internal::governor::speculation_enabled()) { int num_retries = 0; unsigned int abort_code; do { tbb::internal::atomic_backoff backoff; if(this->state) { if(only_speculate) return; do { backoff.pause(); // test the spin_rw_mutex (real readers or writers) } while(this->state); } // _xbegin returns -1 on success or the abort code, so capture it if(( abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) ) { // started speculation #if !__TBB_RW_MUTEX_DELAY_TEST if(this->state) { // add spin_rw_mutex to read-set. // reader or writer grabbed the lock, so abort. __TBB_machine_transaction_conflict_abort(); } #endif s.transaction_state = RTM_transacting_writer; // Don not wrap the following assignment to a function, // because it can abort the transaction in debug. Need mutex for release(). s.my_scoped_lock.mutex = this; return; // successfully started speculation } ++num_retries; } while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_write) ); } if(only_speculate) return; // should apply a real try_lock... s.my_scoped_lock.acquire(*this, true); // kill transactional writers __TBB_ASSERT(!w_flag, "After acquire for write, w_flag already true"); w_flag = true; // kill transactional readers s.transaction_state = RTM_real_writer; return; }