bool GenericTaskQueue<E, F, N>::pop_local_slow(uint localBot, Age oldAge) { // This queue was observed to contain exactly one element; either this // thread will claim it, or a competing "pop_global". In either case, // the queue will be logically empty afterwards. Create a new Age value // that represents the empty queue for the given value of "_bottom". (We // must also increment "tag" because of the case where "bottom == 1", // "top == 0". A pop_global could read the queue element in that case, // then have the owner thread do a pop followed by another push. Without // the incrementing of "tag", the pop_global's CAS could succeed, // allowing it to believe it has claimed the stale element.) Age newAge((idx_t)localBot, oldAge.tag() + 1); // Perhaps a competing pop_global has already incremented "top", in which // case it wins the element. if (localBot == oldAge.top()) { // No competing pop_global has yet incremented "top"; we'll try to // install new_age, thus claiming the element. Age tempAge = _age.cmpxchg(newAge, oldAge); if (tempAge == oldAge) { // We win. assert(dirty_size(localBot, _age.top()) != N - 1, "sanity"); TASKQUEUE_STATS_ONLY(stats.record_pop_slow()); return true; } } // We lose; a completing pop_global gets the element. But the queue is empty // and top is greater than bottom. Fix this representation of the empty queue // to become the canonical one. _age.set(newAge); assert(dirty_size(localBot, _age.top()) != N - 1, "sanity"); return false; }
bool GenericTaskQueue<E, F, N>::pop_global(volatile E& t) { Age oldAge = _age.get(); // Architectures with weak memory model require a barrier here // to guarantee that bottom is not older than age, // which is crucial for the correctness of the algorithm. #if !(defined SPARC || defined IA32 || defined AMD64) OrderAccess::fence(); #endif uint localBot = OrderAccess::load_acquire((volatile juint*)&_bottom); uint n_elems = size(localBot, oldAge.top()); if (n_elems == 0) { return false; } // g++ complains if the volatile result of the assignment is // unused, so we cast the volatile away. We cannot cast directly // to void, because gcc treats that as not using the result of the // assignment. However, casting to E& means that we trigger an // unused-value warning. So, we cast the E& to void. (void) const_cast<E&>(t = _elems[oldAge.top()]); Age newAge(oldAge); newAge.increment(); Age resAge = _age.cmpxchg(newAge, oldAge); // Note that using "_bottom" here might fail, since a pop_local might // have decremented it. assert(dirty_size(localBot, newAge.top()) != N - 1, "sanity"); return resAge == oldAge; }
template<class E, MEMFLAGS F, unsigned int N> inline bool GenericTaskQueue<E, F, N>::pop_local(volatile E& t) { uint localBot = _bottom; // This value cannot be N-1. That can only occur as a result of // the assignment to bottom in this method. If it does, this method // resets the size to 0 before the next call (which is sequential, // since this is pop_local.) uint dirty_n_elems = dirty_size(localBot, _age.top()); assert(dirty_n_elems != N - 1, "Shouldn't be possible..."); if (dirty_n_elems == 0) return false; localBot = decrement_index(localBot); _bottom = localBot; // This is necessary to prevent any read below from being reordered // before the store just above. OrderAccess::fence(); // g++ complains if the volatile result of the assignment is // unused, so we cast the volatile away. We cannot cast directly // to void, because gcc treats that as not using the result of the // assignment. However, casting to E& means that we trigger an // unused-value warning. So, we cast the E& to void. (void) const_cast<E&>(t = _elems[localBot]); // This is a second read of "age"; the "size()" above is the first. // If there's still at least one element in the queue, based on the // "_bottom" and "age" we've read, then there can be no interference with // a "pop_global" operation, and we're done. idx_t tp = _age.top(); // XXX if (size(localBot, tp) > 0) { assert(dirty_size(localBot, tp) != N - 1, "sanity"); TASKQUEUE_STATS_ONLY(stats.record_pop()); return true; } else { // Otherwise, the queue contained exactly one element; we take the slow // path. return pop_local_slow(localBot, _age.get()); } }
// Returns the size corresponding to the given "bot" and "top". uint size(uint bot, uint top) { uint sz = dirty_size(bot, top); // Has the queue "wrapped", so that bottom is less than top? There's a // complicated special case here. A pair of threads could perform pop_local // and pop_global operations concurrently, starting from a state in which // _bottom == _top+1. The pop_local could succeed in decrementing _bottom, // and the pop_global in incrementing _top (in which case the pop_global // will be awarded the contested queue element.) The resulting state must // be interpreted as an empty queue. (We only need to worry about one such // event: only the queue owner performs pop_local's, and several concurrent // threads attempting to perform the pop_global will all perform the same // CAS, and only one can succeed.) Any stealing thread that reads after // either the increment or decrement will see an empty queue, and will not // join the competitors. The "sz == -1 || sz == N-1" state will not be // modified by concurrent queues, so the owner thread can reset the state to // _bottom == top so subsequent pushes will be performed normally. return (sz == N - 1) ? 0 : sz; }
template<class E, MEMFLAGS F, unsigned int N> inline bool GenericTaskQueue<E, F, N>::push(E t) { uint localBot = _bottom; assert(localBot < N, "_bottom out of range."); idx_t top = _age.top(); uint dirty_n_elems = dirty_size(localBot, top); assert(dirty_n_elems < N, "n_elems out of range."); if (dirty_n_elems < max_elems()) { // g++ complains if the volatile result of the assignment is // unused, so we cast the volatile away. We cannot cast directly // to void, because gcc treats that as not using the result of the // assignment. However, casting to E& means that we trigger an // unused-value warning. So, we cast the E& to void. (void) const_cast<E&>(_elems[localBot] = t); OrderAccess::release_store(&_bottom, increment_index(localBot)); TASKQUEUE_STATS_ONLY(stats.record_push()); return true; } else { return push_slow(t, dirty_n_elems); } }