/*
* Remove a waiter from a lock
*
* Must be called with lock->wait_lock held
*/
static void remove_waiter(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
unsigned long flags)
{
int first = (waiter == rt_mutex_top_waiter(lock));
struct task_struct *owner = rt_mutex_owner(lock);
int chain_walk = 0;
raw_spin_lock(¤t->pi_lock);
plist_del(&waiter->list_entry, &lock->wait_list);
current->pi_blocked_on = NULL;
raw_spin_unlock(¤t->pi_lock);
if (!owner) {
BUG_ON(first);
return;
}
if (first) {
raw_spin_lock(&owner->pi_lock);
plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
if (rt_mutex_has_waiters(lock)) {
struct rt_mutex_waiter *next;
next = rt_mutex_top_waiter(lock);
plist_add(&next->pi_list_entry, &owner->pi_waiters);
}
__rt_mutex_adjust_prio(owner);
if (rt_mutex_real_waiter(owner->pi_blocked_on))
chain_walk = 1;
raw_spin_unlock(&owner->pi_lock);
}
WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
if (!chain_walk)
return;
/* gets dropped in rt_mutex_adjust_prio_chain()! */
get_task_struct(owner);
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
raw_spin_lock_irq(&lock->wait_lock);
}
/*
* Wake up the next waiter on the lock.
*
* Remove the top waiter from the current tasks waiter list and from
* the lock waiter list. Set it as pending owner. Then wake it up.
*
* Called with lock->wait_lock held.
*/
static void wakeup_next_waiter(struct rt_mutex *lock)
{
struct rt_mutex_waiter *waiter;
struct task_struct *pendowner;
unsigned long flags;
spin_lock_irqsave(¤t->pi_lock, flags);
waiter = rt_mutex_top_waiter(lock);
plist_del(&waiter->list_entry, &lock->wait_list);
/*
* Remove it from current->pi_waiters. We do not adjust a
* possible priority boost right now. We execute wakeup in the
* boosted mode and go back to normal after releasing
* lock->wait_lock.
*/
plist_del(&waiter->pi_list_entry, ¤t->pi_waiters);
pendowner = waiter->task;
waiter->task = NULL;
rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING);
spin_unlock_irqrestore(¤t->pi_lock, flags);
/*
* Clear the pi_blocked_on variable and enqueue a possible
* waiter into the pi_waiters list of the pending owner. This
* prevents that in case the pending owner gets unboosted a
* waiter with higher priority than pending-owner->normal_prio
* is blocked on the unboosted (pending) owner.
*/
spin_lock_irqsave(&pendowner->pi_lock, flags);
WARN_ON(!pendowner->pi_blocked_on);
WARN_ON(pendowner->pi_blocked_on != waiter);
WARN_ON(pendowner->pi_blocked_on->lock != lock);
pendowner->pi_blocked_on = NULL;
if (rt_mutex_has_waiters(lock)) {
struct rt_mutex_waiter *next;
next = rt_mutex_top_waiter(lock);
plist_add(&next->pi_list_entry, &pendowner->pi_waiters);
}
spin_unlock_irqrestore(&pendowner->pi_lock, flags);
wake_up_process(pendowner);
}
/*
* Remove a waiter from a lock and give up
*
* Must be called with lock->wait_lock held and
* have just failed to try_to_take_rt_mutex().
*/
static void remove_waiter(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter)
{
int first = (waiter == rt_mutex_top_waiter(lock));
struct task_struct *owner = rt_mutex_owner(lock);
unsigned long flags;
int chain_walk = 0;
raw_spin_lock_irqsave(¤t->pi_lock, flags);
rt_mutex_dequeue(lock, waiter);
current->pi_blocked_on = NULL;
raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
if (!owner)
return;
if (first) {
raw_spin_lock_irqsave(&owner->pi_lock, flags);
rt_mutex_dequeue_pi(owner, waiter);
if (rt_mutex_has_waiters(lock)) {
struct rt_mutex_waiter *next;
next = rt_mutex_top_waiter(lock);
rt_mutex_enqueue_pi(owner, next);
}
__rt_mutex_adjust_prio(owner);
if (owner->pi_blocked_on)
chain_walk = 1;
raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
}
if (!chain_walk)
return;
/* gets dropped in rt_mutex_adjust_prio_chain()! */
get_task_struct(owner);
raw_spin_unlock(&lock->wait_lock);
rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
raw_spin_lock(&lock->wait_lock);
}
/*
* Remove a waiter from a lock
*
* Must be called with lock->wait_lock held
*/
static void remove_waiter(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter)
{
int first = (waiter == rt_mutex_top_waiter(lock));
struct task_struct *owner = rt_mutex_owner(lock);
unsigned long flags;
int boost = 0;
spin_lock_irqsave(¤t->pi_lock, flags);
plist_del(&waiter->list_entry, &lock->wait_list);
waiter->task = NULL;
current->pi_blocked_on = NULL;
spin_unlock_irqrestore(¤t->pi_lock, flags);
if (first && owner != current) {
spin_lock_irqsave(&owner->pi_lock, flags);
plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
if (rt_mutex_has_waiters(lock)) {
struct rt_mutex_waiter *next;
next = rt_mutex_top_waiter(lock);
plist_add(&next->pi_list_entry, &owner->pi_waiters);
}
__rt_mutex_adjust_prio(owner);
if (owner->pi_blocked_on) {
boost = 1;
/* gets dropped in rt_mutex_adjust_prio_chain()! */
get_task_struct(owner);
}
spin_unlock_irqrestore(&owner->pi_lock, flags);
}
WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
if (!boost)
return;
spin_unlock(&lock->wait_lock);
rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
spin_lock(&lock->wait_lock);
}
/*
* Slow path to release a rt-mutex:
*/
static void __sched
rt_mutex_slowunlock(struct rt_mutex *lock)
{
raw_spin_lock(&lock->wait_lock);
debug_rt_mutex_unlock(lock);
rt_mutex_deadlock_account_unlock(current);
if (!rt_mutex_has_waiters(lock)) {
lock->owner = NULL;
raw_spin_unlock(&lock->wait_lock);
return;
}
wakeup_next_waiter(lock);
raw_spin_unlock(&lock->wait_lock);
/* Undo pi boosting if necessary: */
rt_mutex_adjust_prio(current);
}
/*
* Slow path to release a rt_mutex spin_lock style
*/
static void fastcall noinline __sched
rt_spin_lock_slowunlock(struct rt_mutex *lock)
{
unsigned long flags;
spin_lock_irqsave(&lock->wait_lock, flags);
debug_rt_mutex_unlock(lock);
rt_mutex_deadlock_account_unlock(current);
if (!rt_mutex_has_waiters(lock)) {
lock->owner = NULL;
spin_unlock_irqrestore(&lock->wait_lock, flags);
return;
}
wakeup_next_waiter(lock, 1);
spin_unlock_irqrestore(&lock->wait_lock, flags);
/* Undo pi boosting.when necessary */
rt_mutex_adjust_prio(current);
}
static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
{
if (!rt_mutex_has_waiters(lock))
clear_rt_mutex_waiters(lock);
}
/*
* Task blocks on lock.
*
* Prepare waiter and propagate pi chain
*
* This must be called with lock->wait_lock held.
*/
static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
int detect_deadlock)
{
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex_waiter *top_waiter = waiter;
unsigned long flags;
int chain_walk = 0, res;
raw_spin_lock_irqsave(&task->pi_lock, flags);
__rt_mutex_adjust_prio(task);
waiter->task = task;
waiter->lock = lock;
plist_node_init(&waiter->list_entry, task->prio);
plist_node_init(&waiter->pi_list_entry, task->prio);
/* Get the top priority waiter on the lock */
if (rt_mutex_has_waiters(lock))
top_waiter = rt_mutex_top_waiter(lock);
plist_add(&waiter->list_entry, &lock->wait_list);
task->pi_blocked_on = waiter;
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
if (!owner)
return 0;
if (waiter == rt_mutex_top_waiter(lock)) {
raw_spin_lock_irqsave(&owner->pi_lock, flags);
plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
__rt_mutex_adjust_prio(owner);
if (owner->pi_blocked_on)
chain_walk = 1;
raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
}
else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
chain_walk = 1;
if (!chain_walk)
return 0;
/*
* The owner can't disappear while holding a lock,
* so the owner struct is protected by wait_lock.
* Gets dropped in rt_mutex_adjust_prio_chain()!
*/
get_task_struct(owner);
raw_spin_unlock(&lock->wait_lock);
res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
task);
raw_spin_lock(&lock->wait_lock);
return res;
}
/*
* Try to take an rt-mutex
*
* Must be called with lock->wait_lock held.
*
* @lock: the lock to be acquired.
* @task: the task which wants to acquire the lock
* @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
*/
static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
struct rt_mutex_waiter *waiter)
{
/*
* We have to be careful here if the atomic speedups are
* enabled, such that, when
* - no other waiter is on the lock
* - the lock has been released since we did the cmpxchg
* the lock can be released or taken while we are doing the
* checks and marking the lock with RT_MUTEX_HAS_WAITERS.
*
* The atomic acquire/release aware variant of
* mark_rt_mutex_waiters uses a cmpxchg loop. After setting
* the WAITERS bit, the atomic release / acquire can not
* happen anymore and lock->wait_lock protects us from the
* non-atomic case.
*
* Note, that this might set lock->owner =
* RT_MUTEX_HAS_WAITERS in the case the lock is not contended
* any more. This is fixed up when we take the ownership.
* This is the transitional state explained at the top of this file.
*/
mark_rt_mutex_waiters(lock);
if (rt_mutex_owner(lock))
return 0;
/*
* It will get the lock because of one of these conditions:
* 1) there is no waiter
* 2) higher priority than waiters
* 3) it is top waiter
*/
if (rt_mutex_has_waiters(lock)) {
if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
if (!waiter || waiter != rt_mutex_top_waiter(lock))
return 0;
}
}
if (waiter || rt_mutex_has_waiters(lock)) {
unsigned long flags;
struct rt_mutex_waiter *top;
raw_spin_lock_irqsave(&task->pi_lock, flags);
/* remove the queued waiter. */
if (waiter) {
plist_del(&waiter->list_entry, &lock->wait_list);
task->pi_blocked_on = NULL;
}
/*
* We have to enqueue the top waiter(if it exists) into
* task->pi_waiters list.
*/
if (rt_mutex_has_waiters(lock)) {
top = rt_mutex_top_waiter(lock);
top->pi_list_entry.prio = top->list_entry.prio;
plist_add(&top->pi_list_entry, &task->pi_waiters);
}
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
}
/* We got the lock. */
debug_rt_mutex_lock(lock);
rt_mutex_set_owner(lock, task);
rt_mutex_deadlock_account_lock(lock, task);
return 1;
}
/*
* Task blocks on lock.
*
* Prepare waiter and propagate pi chain
*
* This must be called with lock->wait_lock held.
*/
static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
int detect_deadlock)
{
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex_waiter *top_waiter = waiter;
unsigned long flags;
int chain_walk = 0, res;
/*
* Early deadlock detection. We really don't want the task to
* enqueue on itself just to untangle the mess later. It's not
* only an optimization. We drop the locks, so another waiter
* can come in before the chain walk detects the deadlock. So
* the other will detect the deadlock and return -EDEADLOCK,
* which is wrong, as the other waiter is not in a deadlock
* situation.
*/
if (detect_deadlock && owner == task)
return -EDEADLK;
raw_spin_lock_irqsave(&task->pi_lock, flags);
__rt_mutex_adjust_prio(task);
waiter->task = task;
waiter->lock = lock;
plist_node_init(&waiter->list_entry, task->prio);
plist_node_init(&waiter->pi_list_entry, task->prio);
/* Get the top priority waiter on the lock */
if (rt_mutex_has_waiters(lock))
top_waiter = rt_mutex_top_waiter(lock);
plist_add(&waiter->list_entry, &lock->wait_list);
task->pi_blocked_on = waiter;
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
if (!owner)
return 0;
if (waiter == rt_mutex_top_waiter(lock)) {
raw_spin_lock_irqsave(&owner->pi_lock, flags);
plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
__rt_mutex_adjust_prio(owner);
if (owner->pi_blocked_on)
chain_walk = 1;
raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
}
else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
chain_walk = 1;
if (!chain_walk)
return 0;
/*
* The owner can't disappear while holding a lock,
* so the owner struct is protected by wait_lock.
* Gets dropped in rt_mutex_adjust_prio_chain()!
*/
get_task_struct(owner);
raw_spin_unlock(&lock->wait_lock);
res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
task);
raw_spin_lock(&lock->wait_lock);
return res;
}
/*
* Slow path lock function spin_lock style: this variant is very
* careful not to miss any non-lock wakeups.
*
* The wakeup side uses wake_up_process_mutex, which, combined with
* the xchg code of this function is a transparent sleep/wakeup
* mechanism nested within any existing sleep/wakeup mechanism. This
* enables the seemless use of arbitrary (blocking) spinlocks within
* sleep/wakeup event loops.
*/
static void noinline __sched
rt_spin_lock_slowlock(struct rt_mutex *lock)
{
struct rt_mutex_waiter waiter;
unsigned long saved_state, flags;
/* orig_owner is only set if next_waiter is set */
struct task_struct *uninitialized_var(orig_owner);
int next_waiter;
int saved_lock_depth;
int ret;
debug_rt_mutex_init_waiter(&waiter);
waiter.task = NULL;
raw_spin_lock_irqsave(&lock->wait_lock, flags);
init_lists(lock);
if (do_try_to_take_rt_mutex(lock, current, NULL, STEAL_LATERAL)) {
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
return;
}
BUG_ON(rt_mutex_owner(lock) == current);
/*
* Here we save whatever state the task was in originally,
* we'll restore it at the end of the function and we'll take
* any intermediate wakeup into account as well, independently
* of the lock sleep/wakeup mechanism. When we get a real
* wakeup the task->state is TASK_RUNNING and we change
* saved_state accordingly. If we did not get a real wakeup
* then we return with the saved state. We need to be careful
* about original state TASK_INTERRUPTIBLE as well, as we
* could miss a wakeup_interruptible()
*/
saved_state = rt_set_current_blocked_state(current->state);
/*
* Prevent schedule() to drop BKL, while waiting for
* the lock ! We restore lock_depth when we come back.
*/
saved_lock_depth = current->lock_depth;
current->lock_depth = -1;
ret = task_blocks_on_rt_mutex(lock, &waiter, current, 0, flags, 1);
BUG_ON(ret);
for (;;) {
int sleep = 1;
/* Try to acquire the lock again. */
if (do_try_to_take_rt_mutex(lock, current, &waiter, STEAL_LATERAL))
break;
next_waiter = &waiter == rt_mutex_top_waiter(lock);
if (next_waiter) {
orig_owner = rt_mutex_owner(lock);
if (orig_owner)
get_task_struct(orig_owner);
}
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
debug_rt_mutex_print_deadlock(&waiter);
if (next_waiter && orig_owner) {
if (!adaptive_wait(&waiter, orig_owner))
sleep = 0;
put_task_struct(orig_owner);
}
if (sleep)
schedule_rt_mutex(lock);
raw_spin_lock_irqsave(&lock->wait_lock, flags);
saved_state = rt_set_current_blocked_state(saved_state);
}
current->lock_depth = saved_lock_depth;
rt_restore_current_state(saved_state);
/*
* try_to_take_rt_mutex() sets the waiter bit
* unconditionally. We might have to fix that up:
*/
fixup_rt_mutex_waiters(lock);
BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock));
BUG_ON(!plist_node_empty(&waiter.list_entry));
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
debug_rt_mutex_free_waiter(&waiter);
}
/*
* Task blocks on lock.
*
* Prepare waiter and propagate pi chain
*
* This must be called with lock->wait_lock held.
*/
static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
int detect_deadlock, unsigned long flags,
int savestate)
{
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex_waiter *top_waiter = waiter;
int chain_walk = 0, res;
raw_spin_lock(&task->pi_lock);
/*
* In the case of futex requeue PI, this will be a proxy
* lock. The task will wake unaware that it is enqueueed on
* this lock. Avoid blocking on two locks and corrupting
* pi_blocked_on via the PI_WAKEUP_INPROGRESS
* flag. futex_wait_requeue_pi() sets this when it wakes up
* before requeue (due to a signal or timeout). Do not enqueue
* the task if PI_WAKEUP_INPROGRESS is set.
*/
if (task != current && task->pi_blocked_on == PI_WAKEUP_INPROGRESS) {
raw_spin_unlock(&task->pi_lock);
return -EAGAIN;
}
BUG_ON(rt_mutex_real_waiter(task->pi_blocked_on));
__rt_mutex_adjust_prio(task);
waiter->task = task;
waiter->lock = lock;
waiter->savestate = savestate;
plist_node_init(&waiter->list_entry, task->prio);
plist_node_init(&waiter->pi_list_entry, task->prio);
/* Get the top priority waiter on the lock */
if (rt_mutex_has_waiters(lock))
top_waiter = rt_mutex_top_waiter(lock);
plist_add(&waiter->list_entry, &lock->wait_list);
task->pi_blocked_on = waiter;
raw_spin_unlock(&task->pi_lock);
if (!owner)
return 0;
if (waiter == rt_mutex_top_waiter(lock)) {
raw_spin_lock(&owner->pi_lock);
plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
__rt_mutex_adjust_prio(owner);
if (rt_mutex_real_waiter(owner->pi_blocked_on))
chain_walk = 1;
raw_spin_unlock(&owner->pi_lock);
}
else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
chain_walk = 1;
if (!chain_walk)
return 0;
/*
* The owner can't disappear while holding a lock,
* so the owner struct is protected by wait_lock.
* Gets dropped in rt_mutex_adjust_prio_chain()!
*/
get_task_struct(owner);
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
task);
raw_spin_lock_irq(&lock->wait_lock);
return res;
}
static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
{
unsigned long owner, *p = (unsigned long *) &lock->owner;
if (rt_mutex_has_waiters(lock))
return;
/*
* The rbtree has no waiters enqueued, now make sure that the
* lock->owner still has the waiters bit set, otherwise the
* following can happen:
*
* CPU 0 CPU 1 CPU2
* l->owner=T1
* rt_mutex_lock(l)
* lock(l->lock)
* l->owner = T1 | HAS_WAITERS;
* enqueue(T2)
* boost()
* unlock(l->lock)
* block()
*
* rt_mutex_lock(l)
* lock(l->lock)
* l->owner = T1 | HAS_WAITERS;
* enqueue(T3)
* boost()
* unlock(l->lock)
* block()
* signal(->T2) signal(->T3)
* lock(l->lock)
* dequeue(T2)
* deboost()
* unlock(l->lock)
* lock(l->lock)
* dequeue(T3)
* ==> wait list is empty
* deboost()
* unlock(l->lock)
* lock(l->lock)
* fixup_rt_mutex_waiters()
* if (wait_list_empty(l) {
* l->owner = owner
* owner = l->owner & ~HAS_WAITERS;
* ==> l->owner = T1
* }
* lock(l->lock)
* rt_mutex_unlock(l) fixup_rt_mutex_waiters()
* if (wait_list_empty(l) {
* owner = l->owner & ~HAS_WAITERS;
* cmpxchg(l->owner, T1, NULL)
* ===> Success (l->owner = NULL)
*
* l->owner = owner
* ==> l->owner = T1
* }
*
* With the check for the waiter bit in place T3 on CPU2 will not
* overwrite. All tasks fiddling with the waiters bit are
* serialized by l->lock, so nothing else can modify the waiters
* bit. If the bit is set then nothing can change l->owner either
* so the simple RMW is safe. The cmpxchg() will simply fail if it
* happens in the middle of the RMW because the waiters bit is
* still set.
*/
owner = READ_ONCE(*p);
if (owner & RT_MUTEX_HAS_WAITERS)
WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);
}
