Example #1
0
    void slowpathLock(Node::Ptr oldTail) {
        Node me;
        Node::Ptr curTail;
        bool newThreads;

        // Step one, put ourselves at the back of the queue
        for (;;) {
            Node::Ptr newTail = Node::Ptr(&me, oldTail.tag());

            // Enqueue ourselves...
            if (tail_.compare_exchange_strong(oldTail, newTail,
                                              std::memory_order_acq_rel,
                                              std::memory_order_relaxed)) break;

            // OK, maybe the whole thing is just unlocked now?
            if (oldTail == Node::Ptr(nullptr, 0)) {
                // If so, try the top level lock
                if (tail_.compare_exchange_strong(oldTail,
                                                  Node::Ptr(nullptr, 1),
                                                  std::memory_order_acquire,
                                                  std::memory_order_relaxed))
                    goto out;
            }
        }

        // Step two: OK, there is an actual queue, so link up with the old
        // tail and wait until we are at the head of the queue
        if (oldTail.ptr()) {
            // * Writing into the oldTail is safe because threads can't
            //   leave unless there is no thread after them or they have
            //   marked the next ready
            oldTail->next.store(&me, std::memory_order_release);

            while (!me.ready.load(std::memory_order_acquire)) delay();
        }

        // Step three: wait until the lock is freed
        while ((curTail = tail_.load(std::memory_order_relaxed)).tag()) {
            delay();
        }

        // Step four: take the lock
        for (;;) {
            assert_eq(curTail.tag(), 0);
            assert_ne(curTail.ptr(), nullptr);

            newThreads = curTail.ptr() != &me;

            // If there aren't any waiters after us, the queue is
            // empty. Otherwise, keep the old tail.
            Node *newTailP = newThreads ? curTail : nullptr;
            Node::Ptr newTail = Node::Ptr(newTailP, 1);

            if (tail_.compare_exchange_strong(curTail, newTail,
                                              std::memory_order_acquire,
                                              std::memory_order_relaxed)) break;
        }

        // Step five: now that we have the lock, if any threads came
        // in after us, indicate to the next one that it is at the
        // head of the queue
        if (newThreads) {
            // Next thread might not have written itself in, yet,
            // so we have to wait.
            Node *next;
            while (!(next = me.next.load(std::memory_order_acquire))) delay();
            next->ready.store(true, std::memory_order_release);
        }
    out:
        return;
    }
Example #2
0
    void slowpathLock(Node::Ptr oldTail) {
        // makes sure that init of me.next is prior to tail_link in
        // other thread
        VEDGE(node_init, enqueue);
        // init of me needs to be done before publishing it to
        // previous thread also
        VEDGE(node_init, tail_link);
        // Can't write self into previous node until previous node inited
        XEDGE(enqueue, tail_link);

        LS(node_init, Node me);
        Node::Ptr curTail;
        bool newThreads;

        // Step one, put ourselves at the back of the queue
        for (;;) {
            Node::Ptr newTail = Node::Ptr(&me, oldTail.tag());

            // Enqueue ourselves...
            if (L(enqueue,
                  tail_.compare_exchange_strong(oldTail, newTail))) break;

            // OK, maybe the whole thing is just unlocked now?
            if (oldTail == Node::Ptr(nullptr, 0)) {
                // If so, try the top level lock
                if (tail_.compare_exchange_strong(oldTail,
                                                  Node::Ptr(nullptr, 1)))
                    goto out;
            }

            delay();
        }

        // Need to make sure not to compete for the lock before the
        // right time. This makes sure the ordering doesn't get messed
        // up.
        XEDGE(ready_wait, lock);

        // Step two: OK, there is an actual queue, so link up with the old
        // tail and wait until we are at the head of the queue
        if (oldTail.ptr()) {
            // * Writing into the oldTail is safe because threads can't
            //   leave unless there is no thread after them or they have
            //   marked the next ready
            L(tail_link, oldTail->next = &me);

            while (!L(ready_wait, me.ready)) delay();
        }

        // Step three: wait until the lock is freed
        // We don't need a a constraint from this load; "lock" serves
        // to handle this just fine: lock can't succeed until we've
        // read an unlocked tail_.
        while ((curTail = tail_).tag()) {
            delay();
        }

        // Our lock acquisition needs to be finished before we give the
        // next thread a chance to try to acquire the lock or it could
        // compete with us for it, causing trouble.
        VEDGE(lock, signal_next);

        // Step four: take the lock
        for (;;) {
            assert_eq(curTail.tag(), 0);
            assert_ne(curTail.ptr(), nullptr);

            newThreads = curTail.ptr() != &me;

            // If there aren't any waiters after us, the queue is
            // empty. Otherwise, keep the old tail.
            Node *newTailP = newThreads ? curTail : nullptr;
            Node::Ptr newTail = Node::Ptr(newTailP, 1);

            if (L(lock, tail_.compare_exchange_strong(curTail, newTail))) break;
        }

        // Step five: now that we have the lock, if any threads came
        // in after us, indicate to the next one that it is at the
        // head of the queue
        if (newThreads) {
            // Next thread might not have written itself in, yet,
            // so we have to wait.
            Node *next;
            XEDGE(load_next, signal_next);
            while (!L(load_next, next = me.next)) delay();
            L(signal_next, next->ready = true);
        }

        //printf("full slowpath out\n");
    out:
        //printf("made it out of slowpath!\n");
        return;
    }