int
rw_enter(struct rwlock *rwl, int flags)
{
const struct rwlock_op *op;
struct sleep_state sls;
unsigned long inc, o;
int error;
op = &rw_ops[flags & RW_OPMASK];
inc = op->inc + RW_PROC(curproc) * op->proc_mult;
retry:
while (__predict_false(((o = rwl->rwl_owner) & op->check) != 0)) {
unsigned long set = o | op->wait_set;
int do_sleep;
rw_enter_diag(rwl, flags);
if (flags & RW_NOSLEEP)
return (EBUSY);
sleep_setup(&sls, rwl, op->wait_prio, rwl->rwl_name);
if (flags & RW_INTR)
sleep_setup_signal(&sls, op->wait_prio | PCATCH);
do_sleep = !rw_cas(&rwl->rwl_owner, o, set);
sleep_finish(&sls, do_sleep);
if ((flags & RW_INTR) &&
(error = sleep_finish_signal(&sls)) != 0)
return (error);
if (flags & RW_SLEEPFAIL)
return (EAGAIN);
}
if (__predict_false(rw_cas(&rwl->rwl_owner, o, o + inc)))
goto retry;
/*
* If old lock had RWLOCK_WAIT and RWLOCK_WRLOCK set, it means we
* downgraded a write lock and had possible read waiter, wake them
* to let them retry the lock.
*/
if (__predict_false((o & (RWLOCK_WRLOCK|RWLOCK_WAIT)) ==
(RWLOCK_WRLOCK|RWLOCK_WAIT)))
wakeup(rwl);
return (0);
}
int
pthread_rwlock_trywrlock(pthread_rwlock_t *ptr)
{
uintptr_t owner, next;
pthread_t self;
if (__predict_false(__uselibcstub))
return __libc_rwlock_trywrlock_stub(ptr);
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
self = pthread__self();
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
if (owner != 0)
return EBUSY;
next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED);
if (owner == next) {
/* Got it! */
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
}
}
int
pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr)
{
uintptr_t owner, next;
if (__predict_false(__uselibcstub))
return __libc_rwlock_tryrdlock_stub(ptr);
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
/*
* Don't get a readlock if there is a writer or if there are waiting
* writers; i.e. prefer writers to readers. This strategy is dictated
* by SUSv3.
*/
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) != 0)
return EBUSY;
next = rw_cas(ptr, owner, owner + RW_READ_INCR);
if (owner == next) {
/* Got it! */
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
}
}
void
rw_enter_write(struct rwlock *rwl)
{
struct proc *p = curproc;
if (__predict_false(rw_cas(&rwl->rwl_owner, 0,
RW_PROC(p) | RWLOCK_WRLOCK)))
rw_enter(rwl, RW_WRITE);
}
/*
* Simple cases that should be in MD code and atomic.
*/
void
rw_enter_read(struct rwlock *rwl)
{
unsigned long owner = rwl->rwl_owner;
if (__predict_false((owner & RWLOCK_WRLOCK) ||
rw_cas(&rwl->rwl_owner, owner, owner + RWLOCK_READ_INCR)))
rw_enter(rwl, RW_READ);
}
void
rw_exit_write(struct rwlock *rwl)
{
unsigned long owner = rwl->rwl_owner;
if (__predict_false((owner & RWLOCK_WAIT) ||
rw_cas(&rwl->rwl_owner, owner, 0)))
rw_exit(rwl);
}
void
rw_exit_read(struct rwlock *rwl)
{
unsigned long owner = rwl->rwl_owner;
if (__predict_false((owner & RWLOCK_WAIT) ||
rw_cas(&rwl->rwl_owner, owner, owner - RWLOCK_READ_INCR)))
rw_exit(rwl);
}
void
rw_exit(struct rwlock *rwl)
{
unsigned long owner = rwl->rwl_owner;
int wrlock = owner & RWLOCK_WRLOCK;
unsigned long set;
do {
owner = rwl->rwl_owner;
if (wrlock)
set = 0;
else
set = (owner - RWLOCK_READ_INCR) &
~(RWLOCK_WAIT|RWLOCK_WRWANT);
} while (rw_cas(&rwl->rwl_owner, owner, set));
if (owner & RWLOCK_WAIT)
wakeup(rwl);
}
/*
* rw_vector_enter:
*
* Acquire a rwlock.
*/
void
rw_vector_enter(krwlock_t *rw, const krw_t op)
{
uintptr_t owner, incr, need_wait, set_wait, curthread, next;
turnstile_t *ts;
int queue;
lwp_t *l;
LOCKSTAT_TIMER(slptime);
LOCKSTAT_TIMER(slpcnt);
LOCKSTAT_TIMER(spintime);
LOCKSTAT_COUNTER(spincnt);
LOCKSTAT_FLAG(lsflag);
l = curlwp;
curthread = (uintptr_t)l;
RW_ASSERT(rw, !cpu_intr_p());
RW_ASSERT(rw, curthread != 0);
RW_WANTLOCK(rw, op);
if (panicstr == NULL) {
LOCKDEBUG_BARRIER(&kernel_lock, 1);
}
/*
* We play a slight trick here. If we're a reader, we want
* increment the read count. If we're a writer, we want to
* set the owner field and whe WRITE_LOCKED bit.
*
* In the latter case, we expect those bits to be zero,
* therefore we can use an add operation to set them, which
* means an add operation for both cases.
*/
if (__predict_true(op == RW_READER)) {
incr = RW_READ_INCR;
set_wait = RW_HAS_WAITERS;
need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
queue = TS_READER_Q;
} else {
RW_DASSERT(rw, op == RW_WRITER);
incr = curthread | RW_WRITE_LOCKED;
set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED;
need_wait = RW_WRITE_LOCKED | RW_THREAD;
queue = TS_WRITER_Q;
}
LOCKSTAT_ENTER(lsflag);
KPREEMPT_DISABLE(curlwp);
for (owner = rw->rw_owner; ;) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & need_wait) == 0) {
next = rw_cas(rw, owner, (owner + incr) &
~RW_WRITE_WANTED);
if (__predict_true(next == owner)) {
/* Got it! */
membar_enter();
break;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
owner = next;
continue;
}
if (__predict_false(panicstr != NULL)) {
kpreempt_enable();
return;
}
if (__predict_false(RW_OWNER(rw) == curthread)) {
rw_abort(rw, __func__, "locking against myself");
}
/*
* If the lock owner is running on another CPU, and
* there are no existing waiters, then spin.
*/
if (rw_oncpu(owner)) {
LOCKSTAT_START_TIMER(lsflag, spintime);
u_int count = SPINLOCK_BACKOFF_MIN;
do {
KPREEMPT_ENABLE(curlwp);
SPINLOCK_BACKOFF(count);
KPREEMPT_DISABLE(curlwp);
owner = rw->rw_owner;
} while (rw_oncpu(owner));
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_COUNT(spincnt, 1);
if ((owner & need_wait) == 0)
continue;
}
/*
* Grab the turnstile chain lock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
ts = turnstile_lookup(rw);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
* Reload rw_owner because turnstile_lookup() may have
* spun on the turnstile chain lock.
*/
owner = rw->rw_owner;
if ((owner & need_wait) == 0 || rw_oncpu(owner)) {
turnstile_exit(rw);
continue;
}
next = rw_cas(rw, owner, owner | set_wait);
if (__predict_false(next != owner)) {
turnstile_exit(rw);
owner = next;
continue;
}
LOCKSTAT_START_TIMER(lsflag, slptime);
turnstile_block(ts, queue, rw, &rw_syncobj);
LOCKSTAT_STOP_TIMER(lsflag, slptime);
LOCKSTAT_COUNT(slpcnt, 1);
/*
* No need for a memory barrier because of context switch.
* If not handed the lock, then spin again.
*/
if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread)
break;
owner = rw->rw_owner;
}
KPREEMPT_ENABLE(curlwp);
LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK |
(op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime);
LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime);
LOCKSTAT_EXIT(lsflag);
RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
(op == RW_READER && RW_COUNT(rw) != 0));
RW_LOCKED(rw, op);
}
static int
pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts)
{
uintptr_t owner, next;
pthread_mutex_t *interlock;
pthread_t self;
int error;
self = pthread__self();
#ifdef ERRORCHECK
if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & RW_THREAD) == 0) {
next = rw_cas(ptr, owner,
(uintptr_t)self | RW_WRITE_LOCKED);
if (owner == next) {
/* Got it! */
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
continue;
}
if ((owner & RW_THREAD) == (uintptr_t)self)
return EDEADLK;
/* If held write locked and no waiters, spin. */
if (pthread__rwlock_spin(owner)) {
while (pthread__rwlock_spin(owner)) {
owner = (uintptr_t)ptr->ptr_owner;
}
next = owner;
continue;
}
/*
* Grab the interlock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
interlock = pthread__hashlock(ptr);
pthread_mutex_lock(interlock);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
*/
next = rw_cas(ptr, owner,
owner | RW_HAS_WAITERS | RW_WRITE_WANTED);
if (owner != next) {
pthread_mutex_unlock(interlock);
continue;
}
/* The waiters bit is set - it's safe to sleep. */
PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep);
self->pt_rwlocked = _RW_WANT_WRITE;
self->pt_sleepobj = &ptr->ptr_wblocked;
self->pt_early = pthread__rwlock_early;
error = pthread__park(self, interlock, &ptr->ptr_wblocked,
ts, 0, &ptr->ptr_wblocked);
/* Did we get the lock? */
if (self->pt_rwlocked == _RW_LOCKED) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
membar_enter();
#endif
return 0;
}
if (error != 0)
return error;
pthread__errorfunc(__FILE__, __LINE__, __func__,
"direct handoff failure");
}
}
