void slowpathLock(Node::Ptr oldTail) {
Node me;
Node::Ptr curTail;
bool newThreads;
// Step one, put ourselves at the back of the queue
for (;;) {
Node::Ptr newTail = Node::Ptr(&me, oldTail.tag());
// Enqueue ourselves...
if (tail_.compare_exchange_strong(oldTail, newTail,
std::memory_order_acq_rel,
std::memory_order_relaxed)) break;
// OK, maybe the whole thing is just unlocked now?
if (oldTail == Node::Ptr(nullptr, 0)) {
// If so, try the top level lock
if (tail_.compare_exchange_strong(oldTail,
Node::Ptr(nullptr, 1),
std::memory_order_acquire,
std::memory_order_relaxed))
goto out;
}
}
// Step two: OK, there is an actual queue, so link up with the old
// tail and wait until we are at the head of the queue
if (oldTail.ptr()) {
// * Writing into the oldTail is safe because threads can't
// leave unless there is no thread after them or they have
// marked the next ready
oldTail->next.store(&me, std::memory_order_release);
while (!me.ready.load(std::memory_order_acquire)) delay();
}
// Step three: wait until the lock is freed
while ((curTail = tail_.load(std::memory_order_relaxed)).tag()) {
delay();
}
// Step four: take the lock
for (;;) {
assert_eq(curTail.tag(), 0);
assert_ne(curTail.ptr(), nullptr);
newThreads = curTail.ptr() != &me;
// If there aren't any waiters after us, the queue is
// empty. Otherwise, keep the old tail.
Node *newTailP = newThreads ? curTail : nullptr;
Node::Ptr newTail = Node::Ptr(newTailP, 1);
if (tail_.compare_exchange_strong(curTail, newTail,
std::memory_order_acquire,
std::memory_order_relaxed)) break;
}
// Step five: now that we have the lock, if any threads came
// in after us, indicate to the next one that it is at the
// head of the queue
if (newThreads) {
// Next thread might not have written itself in, yet,
// so we have to wait.
Node *next;
while (!(next = me.next.load(std::memory_order_acquire))) delay();
next->ready.store(true, std::memory_order_release);
}
out:
return;
}
void slowpathLock(Node::Ptr oldTail) {
// makes sure that init of me.next is prior to tail_link in
// other thread
VEDGE(node_init, enqueue);
// init of me needs to be done before publishing it to
// previous thread also
VEDGE(node_init, tail_link);
// Can't write self into previous node until previous node inited
XEDGE(enqueue, tail_link);
LS(node_init, Node me);
Node::Ptr curTail;
bool newThreads;
// Step one, put ourselves at the back of the queue
for (;;) {
Node::Ptr newTail = Node::Ptr(&me, oldTail.tag());
// Enqueue ourselves...
if (L(enqueue,
tail_.compare_exchange_strong(oldTail, newTail))) break;
// OK, maybe the whole thing is just unlocked now?
if (oldTail == Node::Ptr(nullptr, 0)) {
// If so, try the top level lock
if (tail_.compare_exchange_strong(oldTail,
Node::Ptr(nullptr, 1)))
goto out;
}
delay();
}
// Need to make sure not to compete for the lock before the
// right time. This makes sure the ordering doesn't get messed
// up.
XEDGE(ready_wait, lock);
// Step two: OK, there is an actual queue, so link up with the old
// tail and wait until we are at the head of the queue
if (oldTail.ptr()) {
// * Writing into the oldTail is safe because threads can't
// leave unless there is no thread after them or they have
// marked the next ready
L(tail_link, oldTail->next = &me);
while (!L(ready_wait, me.ready)) delay();
}
// Step three: wait until the lock is freed
// We don't need a a constraint from this load; "lock" serves
// to handle this just fine: lock can't succeed until we've
// read an unlocked tail_.
while ((curTail = tail_).tag()) {
delay();
}
// Our lock acquisition needs to be finished before we give the
// next thread a chance to try to acquire the lock or it could
// compete with us for it, causing trouble.
VEDGE(lock, signal_next);
// Step four: take the lock
for (;;) {
assert_eq(curTail.tag(), 0);
assert_ne(curTail.ptr(), nullptr);
newThreads = curTail.ptr() != &me;
// If there aren't any waiters after us, the queue is
// empty. Otherwise, keep the old tail.
Node *newTailP = newThreads ? curTail : nullptr;
Node::Ptr newTail = Node::Ptr(newTailP, 1);
if (L(lock, tail_.compare_exchange_strong(curTail, newTail))) break;
}
// Step five: now that we have the lock, if any threads came
// in after us, indicate to the next one that it is at the
// head of the queue
if (newThreads) {
// Next thread might not have written itself in, yet,
// so we have to wait.
Node *next;
XEDGE(load_next, signal_next);
while (!L(load_next, next = me.next)) delay();
L(signal_next, next->ready = true);
}
//printf("full slowpath out\n");
out:
//printf("made it out of slowpath!\n");
return;
}