template<class T, MEMFLAGS F> bool
GenericTaskQueueSet<T, F>::steal(uint queue_num, int* seed, E& t) {
for (uint i = 0; i < 2 * _n; i++) {
if (steal_best_of_2(queue_num, seed, t)) {
TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(true));
return true;
}
}
TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(false));
return false;
}
bool GenericTaskQueue<E, F, N>::pop_local_slow(uint localBot, Age oldAge) {
// This queue was observed to contain exactly one element; either this
// thread will claim it, or a competing "pop_global". In either case,
// the queue will be logically empty afterwards. Create a new Age value
// that represents the empty queue for the given value of "_bottom". (We
// must also increment "tag" because of the case where "bottom == 1",
// "top == 0". A pop_global could read the queue element in that case,
// then have the owner thread do a pop followed by another push. Without
// the incrementing of "tag", the pop_global's CAS could succeed,
// allowing it to believe it has claimed the stale element.)
Age newAge((idx_t)localBot, oldAge.tag() + 1);
// Perhaps a competing pop_global has already incremented "top", in which
// case it wins the element.
if (localBot == oldAge.top()) {
// No competing pop_global has yet incremented "top"; we'll try to
// install new_age, thus claiming the element.
Age tempAge = _age.cmpxchg(newAge, oldAge);
if (tempAge == oldAge) {
// We win.
assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
TASKQUEUE_STATS_ONLY(stats.record_pop_slow());
return true;
}
}
// We lose; a completing pop_global gets the element. But the queue is empty
// and top is greater than bottom. Fix this representation of the empty queue
// to become the canonical one.
_age.set(newAge);
assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
return false;
}
inline bool OverflowTaskQueue<E, F, N>::push(E t)
{
if (!taskqueue_t::push(t)) {
overflow_stack()->push(t);
TASKQUEUE_STATS_ONLY(stats.record_overflow(overflow_stack()->size()));
}
return true;
}
bool GenericTaskQueue<E, F, N>::push_slow(E t, uint dirty_n_elems) {
if (dirty_n_elems == N - 1) {
// Actually means 0, so do the push.
uint localBot = _bottom;
// g++ complains if the volatile result of the assignment is
// unused, so we cast the volatile away. We cannot cast directly
// to void, because gcc treats that as not using the result of the
// assignment. However, casting to E& means that we trigger an
// unused-value warning. So, we cast the E& to void.
(void)const_cast<E&>(_elems[localBot] = t);
OrderAccess::release_store(&_bottom, increment_index(localBot));
TASKQUEUE_STATS_ONLY(stats.record_push());
return true;
}
return false;
}
template<class E, MEMFLAGS F, unsigned int N> inline bool
GenericTaskQueue<E, F, N>::push(E t) {
uint localBot = _bottom;
assert(localBot < N, "_bottom out of range.");
idx_t top = _age.top();
uint dirty_n_elems = dirty_size(localBot, top);
assert(dirty_n_elems < N, "n_elems out of range.");
if (dirty_n_elems < max_elems()) {
// g++ complains if the volatile result of the assignment is
// unused, so we cast the volatile away. We cannot cast directly
// to void, because gcc treats that as not using the result of the
// assignment. However, casting to E& means that we trigger an
// unused-value warning. So, we cast the E& to void.
(void) const_cast<E&>(_elems[localBot] = t);
OrderAccess::release_store(&_bottom, increment_index(localBot));
TASKQUEUE_STATS_ONLY(stats.record_push());
return true;
} else {
return push_slow(t, dirty_n_elems);
}
}
void StealTask::do_it(GCTaskManager* manager, uint which) {
assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc");
PSPromotionManager* pm =
PSPromotionManager::gc_thread_promotion_manager(which);
pm->drain_stacks(true);
guarantee(pm->stacks_empty(),
"stacks should be empty at this point");
int random_seed = 17;
while(true) {
StarTask p;
if (PSPromotionManager::steal_depth(which, &random_seed, p)) {
TASKQUEUE_STATS_ONLY(pm->record_steal(p));
pm->process_popped_location_depth(p);
pm->drain_stacks_depth(true);
} else {
if (terminator()->offer_termination()) {
break;
}
}
}
guarantee(pm->stacks_empty(), "stacks should be empty at this point");
}
template<class E, MEMFLAGS F, unsigned int N> inline bool
GenericTaskQueue<E, F, N>::pop_local(volatile E& t) {
uint localBot = _bottom;
// This value cannot be N-1. That can only occur as a result of
// the assignment to bottom in this method. If it does, this method
// resets the size to 0 before the next call (which is sequential,
// since this is pop_local.)
uint dirty_n_elems = dirty_size(localBot, _age.top());
assert(dirty_n_elems != N - 1, "Shouldn't be possible...");
if (dirty_n_elems == 0) return false;
localBot = decrement_index(localBot);
_bottom = localBot;
// This is necessary to prevent any read below from being reordered
// before the store just above.
OrderAccess::fence();
// g++ complains if the volatile result of the assignment is
// unused, so we cast the volatile away. We cannot cast directly
// to void, because gcc treats that as not using the result of the
// assignment. However, casting to E& means that we trigger an
// unused-value warning. So, we cast the E& to void.
(void) const_cast<E&>(t = _elems[localBot]);
// This is a second read of "age"; the "size()" above is the first.
// If there's still at least one element in the queue, based on the
// "_bottom" and "age" we've read, then there can be no interference with
// a "pop_global" operation, and we're done.
idx_t tp = _age.top(); // XXX
if (size(localBot, tp) > 0) {
assert(dirty_size(localBot, tp) != N - 1, "sanity");
TASKQUEUE_STATS_ONLY(stats.record_pop());
return true;
} else {
// Otherwise, the queue contained exactly one element; we take the slow
// path.
return pop_local_slow(localBot, _age.get());
}
}