void xnsynch_renice_sleeper(xnthread_t *thread)
{
xnsynch_t *synch = thread->wchan;
xnthread_t *owner;
if (!testbits(synch->status, XNSYNCH_PRIO))
return;
removepq(&synch->pendq, &thread->plink);
insertpqf(&synch->pendq, &thread->plink, thread->cprio);
owner = synch->owner;
if (owner != NULL && thread->cprio > owner->cprio) {
/* The new priority of the sleeping thread is higher
* than the priority of the current owner of the
* resource: we need to update the PI state. */
if (testbits(synch->status, XNSYNCH_CLAIMED)) {
/* The resource is already claimed, just
reorder the claim queue. */
removepq(&owner->claimq, &synch->link);
insertpqf(&owner->claimq, &synch->link, thread->cprio);
} else {
/* The resource was NOT claimed, claim it now
* and boost the owner. */
__setbits(synch->status, XNSYNCH_CLAIMED);
insertpqf(&owner->claimq, &synch->link, thread->cprio);
owner->bprio = owner->cprio;
xnthread_set_state(owner, XNBOOST);
}
/* Renice the owner thread, progressing in the PI
chain as needed. */
xnsynch_renice_thread(owner, thread->cprio);
}
}
void xnsched_weak_setparam(struct xnthread *thread,
const union xnsched_policy_param *p)
{
thread->cprio = p->weak.prio;
if (!xnthread_test_state(thread, XNBOOST))
xnthread_set_state(thread, XNWEAK);
}
/* Must be called with nklock locked, interrupts off. thread must be
* runnable. */
void xnsched_migrate(struct xnthread *thread, struct xnsched *sched)
{
struct xnsched_class *sched_class = thread->sched_class;
if (xnthread_test_state(thread, XNREADY)) {
xnsched_dequeue(thread);
xnthread_clear_state(thread, XNREADY);
}
if (sched_class->sched_migrate)
sched_class->sched_migrate(thread, sched);
/*
* WARNING: the scheduling class may have just changed as a
* result of calling the per-class migration hook.
*/
xnsched_set_resched(thread->sched);
thread->sched = sched;
#ifdef CONFIG_XENO_HW_UNLOCKED_SWITCH
/*
* Mark the thread in flight, xnsched_finish_unlocked_switch()
* will put the thread on the remote runqueue.
*/
xnthread_set_state(thread, XNMIGRATE);
#else /* !CONFIG_XENO_HW_UNLOCKED_SWITCH */
/* Move thread to the remote runnable queue. */
xnsched_putback(thread);
#endif /* !CONFIG_XENO_HW_UNLOCKED_SWITCH */
}
/* NOTE: caller must provide locking */
void xnthread_prepare_wait(struct xnthread_wait_context *wc)
{
struct xnthread *curr = xnpod_current_thread();
curr->wcontext = wc;
wc->oldstate = xnthread_test_state(curr, XNDEFCAN);
xnthread_set_state(curr, XNDEFCAN);
}
/*
* Must be called with nklock locked, interrupts off. Thread may be
* blocked.
*/
void xnsched_migrate_passive(struct xnthread *thread, struct xnsched *sched)
{
migrate_thread(thread, sched);
if (!xnthread_test_state(thread, XNTHREAD_BLOCK_BITS)) {
xnsched_requeue(thread);
xnthread_set_state(thread, XNREADY);
}
}
void ___xnsched_lock(struct xnsched *sched)
{
struct xnthread *curr = sched->curr;
if (curr->lock_count++ == 0) {
sched->lflags |= XNINLOCK;
xnthread_set_state(curr, XNLOCK);
}
}
/*
* Detect when a thread is about to sleep on a synchronization
* object currently owned by someone running in secondary mode.
*/
void xnsynch_detect_relaxed_owner(struct xnsynch *synch, struct xnthread *sleeper)
{
if (xnthread_test_state(sleeper, XNTRAPSW|XNSWREP) == XNTRAPSW &&
xnthread_test_state(synch->owner, XNRELAX)) {
xnthread_set_state(sleeper, XNSWREP);
xnshadow_send_sig(sleeper, SIGDEBUG,
SIGDEBUG_MIGRATE_PRIOINV, 1);
} else
xnthread_clear_state(sleeper, XNSWREP);
}
/* Must be called with nklock locked, interrupts off. */
void xnsched_putback(struct xnthread *thread)
{
if (xnthread_test_state(thread, XNREADY))
xnsched_dequeue(thread);
else
xnthread_set_state(thread, XNREADY);
xnsched_enqueue(thread);
xnsched_set_resched(thread->sched);
}
/* Must be called with nklock locked, interrupts off. */
struct xnthread *xnsched_pick_next(struct xnsched *sched)
{
struct xnthread *curr = sched->curr;
struct xnsched_class *p;
struct xnthread *thread;
if (!xnthread_test_state(curr, XNTHREAD_BLOCK_BITS | XNZOMBIE)) {
/*
* Do not preempt the current thread if it holds the
* scheduler lock.
*/
if (xnthread_test_state(curr, XNLOCK)) {
xnsched_set_self_resched(sched);
return curr;
}
/*
* Push the current thread back to the runnable queue
* of the scheduling class it belongs to, if not yet
* linked to it (XNREADY tells us if it is).
*/
if (!xnthread_test_state(curr, XNREADY)) {
xnsched_requeue(curr);
xnthread_set_state(curr, XNREADY);
}
#ifdef __XENO_SIM__
if (nkpod->schedhook)
nkpod->schedhook(curr, XNREADY);
#endif /* __XENO_SIM__ */
}
/*
* Find the runnable thread having the highest priority among
* all scheduling classes, scanned by decreasing priority.
*/
#ifdef CONFIG_XENO_OPT_SCHED_CLASSES
for_each_xnsched_class(p) {
thread = p->sched_pick(sched);
if (thread) {
xnthread_clear_state(thread, XNREADY);
return thread;
}
}
return NULL; /* Never executed because of the idle class. */
#else /* !CONFIG_XENO_OPT_SCHED_CLASSES */
thread = __xnsched_rt_pick(sched); (void)p;
if (unlikely(thread == NULL))
thread = &sched->rootcb;
xnthread_clear_state(thread, XNREADY);
return thread;
#endif /* CONFIG_XENO_OPT_SCHED_CLASSES */
}
/*
* Must be called with nklock locked, interrupts off. thread must be
* runnable.
*/
void xnsched_migrate(struct xnthread *thread, struct xnsched *sched)
{
migrate_thread(thread, sched);
#ifdef CONFIG_XENO_ARCH_UNLOCKED_SWITCH
/*
* Mark the thread in flight, xnsched_finish_unlocked_switch()
* will put the thread on the remote runqueue.
*/
xnthread_set_state(thread, XNMIGRATE);
#else /* !CONFIG_XENO_ARCH_UNLOCKED_SWITCH */
/* Move thread to the remote runnable queue. */
xnsched_putback(thread);
#endif /* !CONFIG_XENO_ARCH_UNLOCKED_SWITCH */
}
void xnsynch_requeue_sleeper(struct xnthread *thread)
{
struct xnsynch *synch = thread->wchan;
struct xnthread *owner;
if (!testbits(synch->status, XNSYNCH_PRIO))
return;
removepq(&synch->pendq, &thread->plink);
insertpqf(&synch->pendq, &thread->plink, w_cprio(thread));
owner = synch->owner;
if (owner != NULL && w_cprio(thread) > w_cprio(owner)) {
/*
* The new (weighted) priority of the sleeping thread
* is higher than the priority of the current owner of
* the resource: we need to update the PI state.
*/
if (testbits(synch->status, XNSYNCH_CLAIMED)) {
/*
* The resource is already claimed, just
* reorder the claim queue.
*/
removepq(&owner->claimq, &synch->link);
insertpqf(&owner->claimq, &synch->link,
w_cprio(thread));
} else {
/*
* The resource was NOT claimed, claim it now
* and boost the owner.
*/
__setbits(synch->status, XNSYNCH_CLAIMED);
insertpqf(&owner->claimq, &synch->link,
w_cprio(thread));
if (!xnthread_test_state(owner, XNBOOST)) {
owner->bprio = owner->cprio;
xnthread_set_state(owner, XNBOOST);
}
}
/*
* Renice the owner thread, progressing in the PI
* chain as needed.
*/
xnsynch_renice_thread(owner, thread);
}
}
/*
* Detect when a thread is about to relax while holding a
* synchronization object currently claimed by another thread, which
* bears the TWARNSW bit (thus advertising a concern about potential
* spurious relaxes and priority inversion). By relying on the claim
* queue, we restrict the checks to PIP-enabled objects, but that
* already covers most of the use cases anyway.
*/
void xnsynch_detect_claimed_relax(struct xnthread *owner)
{
struct xnpholder *hs, *ht;
struct xnthread *sleeper;
struct xnsynch *synch;
for (hs = getheadpq(&owner->claimq); hs != NULL;
hs = nextpq(&owner->claimq, hs)) {
synch = link2synch(hs);
for (ht = getheadpq(&synch->pendq); ht != NULL;
ht = nextpq(&synch->pendq, ht)) {
sleeper = link2thread(ht, plink);
if (xnthread_test_state(sleeper, XNTRAPSW)) {
xnthread_set_state(sleeper, XNSWREP);
xnshadow_send_sig(sleeper, SIGDEBUG,
SIGDEBUG_MIGRATE_PRIOINV, 1);
}
}
}
}
/* Must be called with nklock locked, interrupts off. thread may be
* blocked. */
void xnsched_migrate_passive(struct xnthread *thread, struct xnsched *sched)
{
struct xnsched_class *sched_class = thread->sched_class;
if (xnthread_test_state(thread, XNREADY)) {
xnsched_dequeue(thread);
xnthread_clear_state(thread, XNREADY);
}
if (sched_class->sched_migrate)
sched_class->sched_migrate(thread, sched);
/*
* WARNING: the scheduling class may have just changed as a
* result of calling the per-class migration hook.
*/
xnsched_set_resched(thread->sched);
thread->sched = sched;
if (!xnthread_test_state(thread, XNTHREAD_BLOCK_BITS)) {
xnsched_requeue(thread);
xnthread_set_state(thread, XNREADY);
}
}
xnflags_t xnsynch_acquire(struct xnsynch *synch, xnticks_t timeout,
xntmode_t timeout_mode)
{
struct xnthread *thread = xnpod_current_thread(), *owner;
xnhandle_t threadh = xnthread_handle(thread), fastlock, old;
const int use_fastlock = xnsynch_fastlock_p(synch);
spl_t s;
XENO_BUGON(NUCLEUS, !testbits(synch->status, XNSYNCH_OWNER));
trace_mark(xn_nucleus, synch_acquire, "synch %p", synch);
redo:
if (use_fastlock) {
xnarch_atomic_t *lockp = xnsynch_fastlock(synch);
fastlock = xnarch_atomic_cmpxchg(lockp,
XN_NO_HANDLE, threadh);
if (likely(fastlock == XN_NO_HANDLE)) {
if (xnthread_test_state(thread, XNOTHER))
xnthread_inc_rescnt(thread);
xnthread_clear_info(thread,
XNRMID | XNTIMEO | XNBREAK);
return 0;
}
xnlock_get_irqsave(&nklock, s);
/* Set claimed bit.
In case it appears to be set already, re-read its state
under nklock so that we don't miss any change between the
lock-less read and here. But also try to avoid cmpxchg
where possible. Only if it appears not to be set, start
with cmpxchg directly. */
if (xnsynch_fast_is_claimed(fastlock)) {
old = xnarch_atomic_get(lockp);
goto test_no_owner;
}
do {
old = xnarch_atomic_cmpxchg(lockp, fastlock,
xnsynch_fast_set_claimed(fastlock, 1));
if (likely(old == fastlock))
break;
test_no_owner:
if (old == XN_NO_HANDLE) {
/* Owner called xnsynch_release
(on another cpu) */
xnlock_put_irqrestore(&nklock, s);
goto redo;
}
fastlock = old;
} while (!xnsynch_fast_is_claimed(fastlock));
owner = xnthread_lookup(xnsynch_fast_mask_claimed(fastlock));
if (!owner) {
/* The handle is broken, therefore pretend that the synch
object was deleted to signal an error. */
xnthread_set_info(thread, XNRMID);
goto unlock_and_exit;
}
xnsynch_set_owner(synch, owner);
} else {
xnlock_get_irqsave(&nklock, s);
owner = synch->owner;
if (!owner) {
synch->owner = thread;
if (xnthread_test_state(thread, XNOTHER))
xnthread_inc_rescnt(thread);
xnthread_clear_info(thread,
XNRMID | XNTIMEO | XNBREAK);
goto unlock_and_exit;
}
}
xnsynch_detect_relaxed_owner(synch, thread);
if (!testbits(synch->status, XNSYNCH_PRIO)) /* i.e. FIFO */
appendpq(&synch->pendq, &thread->plink);
else if (w_cprio(thread) > w_cprio(owner)) {
if (xnthread_test_info(owner, XNWAKEN) && owner->wwake == synch) {
/* Ownership is still pending, steal the resource. */
synch->owner = thread;
xnthread_clear_info(thread, XNRMID | XNTIMEO | XNBREAK);
xnthread_set_info(owner, XNROBBED);
goto grab_and_exit;
}
insertpqf(&synch->pendq, &thread->plink, w_cprio(thread));
if (testbits(synch->status, XNSYNCH_PIP)) {
if (!xnthread_test_state(owner, XNBOOST)) {
owner->bprio = owner->cprio;
xnthread_set_state(owner, XNBOOST);
}
if (testbits(synch->status, XNSYNCH_CLAIMED))
removepq(&owner->claimq, &synch->link);
else
__setbits(synch->status, XNSYNCH_CLAIMED);
insertpqf(&owner->claimq, &synch->link, w_cprio(thread));
xnsynch_renice_thread(owner, thread);
}
} else
insertpqf(&synch->pendq, &thread->plink, w_cprio(thread));
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
thread->wwake = NULL;
xnthread_clear_info(thread, XNWAKEN);
if (xnthread_test_info(thread, XNRMID | XNTIMEO | XNBREAK))
goto unlock_and_exit;
if (xnthread_test_info(thread, XNROBBED)) {
/* Somebody stole us the ownership while we were ready
to run, waiting for the CPU: we need to wait again
for the resource. */
if (timeout_mode != XN_RELATIVE || timeout == XN_INFINITE) {
xnlock_put_irqrestore(&nklock, s);
goto redo;
}
timeout = xntimer_get_timeout_stopped(&thread->rtimer);
if (timeout > 1) { /* Otherwise, it's too late. */
xnlock_put_irqrestore(&nklock, s);
goto redo;
}
xnthread_set_info(thread, XNTIMEO);
} else {
grab_and_exit:
if (xnthread_test_state(thread, XNOTHER))
xnthread_inc_rescnt(thread);
if (use_fastlock) {
xnarch_atomic_t *lockp = xnsynch_fastlock(synch);
/* We are the new owner, update the fastlock
accordingly. */
if (xnsynch_pended_p(synch))
threadh =
xnsynch_fast_set_claimed(threadh, 1);
xnarch_atomic_set(lockp, threadh);
}
}
unlock_and_exit:
xnlock_put_irqrestore(&nklock, s);
return xnthread_test_info(thread, XNRMID|XNTIMEO|XNBREAK);
}
void xnsynch_sleep_on(xnsynch_t *synch, xnticks_t timeout,
xntmode_t timeout_mode)
{
xnthread_t *thread = xnpod_current_thread(), *owner;
spl_t s;
xnlock_get_irqsave(&nklock, s);
trace_mark(xn_nucleus_synch_sleepon,
"thread %p thread_name %s synch %p",
thread, xnthread_name(thread), synch);
if (!testbits(synch->status, XNSYNCH_PRIO)) { /* i.e. FIFO */
appendpq(&synch->pendq, &thread->plink);
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
goto unlock_and_exit;
}
if (!testbits(synch->status, XNSYNCH_PIP)) { /* i.e. no ownership */
insertpqf(&synch->pendq, &thread->plink, thread->cprio);
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
goto unlock_and_exit;
}
redo:
owner = synch->owner;
if (!owner) {
synch->owner = thread;
xnthread_clear_info(thread, XNRMID | XNTIMEO | XNBREAK);
goto unlock_and_exit;
}
if (thread->cprio > owner->cprio) {
if (xnthread_test_info(owner, XNWAKEN) && owner->wwake == synch) {
/* Ownership is still pending, steal the resource. */
synch->owner = thread;
xnthread_clear_info(thread, XNRMID | XNTIMEO | XNBREAK);
xnthread_set_info(owner, XNROBBED);
goto unlock_and_exit;
}
if (!xnthread_test_state(owner, XNBOOST)) {
owner->bprio = owner->cprio;
xnthread_set_state(owner, XNBOOST);
}
if (testbits(synch->status, XNSYNCH_CLAIMED))
removepq(&owner->claimq, &synch->link);
else
__setbits(synch->status, XNSYNCH_CLAIMED);
insertpqf(&owner->claimq, &synch->link, thread->cprio);
insertpqf(&synch->pendq, &thread->plink, thread->cprio);
xnsynch_renice_thread(owner, thread->cprio);
} else
insertpqf(&synch->pendq, &thread->plink, thread->cprio);
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
if (xnthread_test_info(thread, XNRMID | XNTIMEO | XNBREAK))
goto unlock_and_exit;
if (xnthread_test_info(thread, XNROBBED)) {
/* Somebody stole us the ownership while we were ready
to run, waiting for the CPU: we need to wait again
for the resource. */
if (timeout_mode != XN_RELATIVE || timeout == XN_INFINITE)
goto redo;
timeout = xntimer_get_timeout_stopped(&thread->rtimer);
if (timeout > 1) /* Otherwise, it's too late. */
goto redo;
xnthread_set_info(thread, XNTIMEO);
}
unlock_and_exit:
thread->wwake = NULL;
xnthread_clear_info(thread, XNWAKEN);
xnlock_put_irqrestore(&nklock, s);
}
