int xnsynch_flush(struct xnsynch *synch, xnflags_t reason)
{
struct xnpholder *holder;
int status;
spl_t s;
xnlock_get_irqsave(&nklock, s);
trace_mark(xn_nucleus, synch_flush, "synch %p reason %lu",
synch, reason);
status = emptypq_p(&synch->pendq) ? XNSYNCH_DONE : XNSYNCH_RESCHED;
while ((holder = getpq(&synch->pendq)) != NULL) {
struct xnthread *sleeper = link2thread(holder, plink);
xnthread_set_info(sleeper, reason);
sleeper->wchan = NULL;
xnpod_resume_thread(sleeper, XNPEND);
}
if (testbits(synch->status, XNSYNCH_CLAIMED)) {
xnsynch_clear_boost(synch, synch->owner);
status = XNSYNCH_RESCHED;
}
xnlock_put_irqrestore(&nklock, s);
xnarch_post_graph_if(synch, 0, emptypq_p(&synch->pendq));
return status;
}
xnpholder_t *xnsynch_wakeup_this_sleeper(xnsynch_t *synch, xnpholder_t *holder)
{
xnthread_t *thread, *lastowner;
xnpholder_t *nholder;
spl_t s;
xnlock_get_irqsave(&nklock, s);
lastowner = synch->owner;
nholder = poppq(&synch->pendq, holder);
thread = link2thread(holder, plink);
thread->wchan = NULL;
thread->wwake = synch;
synch->owner = thread;
xnthread_set_info(thread, XNWAKEN);
trace_mark(xn_nucleus_synch_wakeup_all,
"thread %p thread_name %s synch %p",
thread, xnthread_name(thread), synch);
xnpod_resume_thread(thread, XNPEND);
if (testbits(synch->status, XNSYNCH_CLAIMED))
xnsynch_clear_boost(synch, lastowner);
xnlock_put_irqrestore(&nklock, s);
xnarch_post_graph_if(synch, 0, emptypq_p(&synch->pendq));
return nholder;
}
static struct xnthread *
xnsynch_release_thread(struct xnsynch *synch, struct xnthread *lastowner)
{
const int use_fastlock = xnsynch_fastlock_p(synch);
xnhandle_t lastownerh, newownerh;
struct xnthread *newowner;
struct xnpholder *holder;
spl_t s;
XENO_BUGON(NUCLEUS, !testbits(synch->status, XNSYNCH_OWNER));
#ifdef CONFIG_XENO_OPT_PERVASIVE
if (xnthread_test_state(lastowner, XNOTHER)) {
if (xnthread_get_rescnt(lastowner) == 0)
xnshadow_send_sig(lastowner, SIGDEBUG,
SIGDEBUG_MIGRATE_PRIOINV, 1);
else
xnthread_dec_rescnt(lastowner);
}
#endif
lastownerh = xnthread_handle(lastowner);
if (use_fastlock &&
likely(xnsynch_fast_release(xnsynch_fastlock(synch), lastownerh)))
return NULL;
xnlock_get_irqsave(&nklock, s);
trace_mark(xn_nucleus, synch_release, "synch %p", synch);
holder = getpq(&synch->pendq);
if (holder) {
newowner = link2thread(holder, plink);
newowner->wchan = NULL;
newowner->wwake = synch;
synch->owner = newowner;
xnthread_set_info(newowner, XNWAKEN);
xnpod_resume_thread(newowner, XNPEND);
if (testbits(synch->status, XNSYNCH_CLAIMED))
xnsynch_clear_boost(synch, lastowner);
newownerh = xnsynch_fast_set_claimed(xnthread_handle(newowner),
xnsynch_pended_p(synch));
} else {
newowner = NULL;
synch->owner = NULL;
newownerh = XN_NO_HANDLE;
}
if (use_fastlock) {
xnarch_atomic_t *lockp = xnsynch_fastlock(synch);
xnarch_atomic_set(lockp, newownerh);
}
xnlock_put_irqrestore(&nklock, s);
xnarch_post_graph_if(synch, 0, emptypq_p(&synch->pendq));
return newowner;
}
static void xnsched_watchdog_handler(struct xntimer *timer)
{
struct xnsched *sched = xnpod_current_sched();
struct xnthread *thread = sched->curr;
if (likely(xnthread_test_state(thread, XNROOT))) {
xnsched_reset_watchdog(sched);
return;
}
if (likely(++sched->wdcount < wd_timeout_arg))
return;
#ifdef CONFIG_XENO_OPT_PERVASIVE
if (xnthread_test_state(thread, XNSHADOW) &&
!xnthread_amok_p(thread)) {
trace_mark(xn_nucleus, watchdog_signal,
"thread %p thread_name %s",
thread, xnthread_name(thread));
xnprintf("watchdog triggered -- signaling runaway thread "
"'%s'\n", xnthread_name(thread));
xnthread_set_info(thread, XNAMOK | XNKICKED);
xnshadow_send_sig(thread, SIGDEBUG, SIGDEBUG_WATCHDOG, 1);
} else
#endif /* CONFIG_XENO_OPT_PERVASIVE */
{
trace_mark(xn_nucleus, watchdog, "thread %p thread_name %s",
thread, xnthread_name(thread));
xnprintf("watchdog triggered -- killing runaway thread '%s'\n",
xnthread_name(thread));
xnpod_delete_thread(thread);
}
xnsched_reset_watchdog(sched);
}
/*
* Detect when a thread is about to sleep on a synchronization
* object currently owned by someone running in secondary mode.
*/
void xnsynch_detect_relaxed_owner(struct xnsynch *synch, struct xnthread *sleeper)
{
if (xnthread_test_state(sleeper, XNTRAPSW) &&
!xnthread_test_info(sleeper, XNSWREP) &&
xnthread_test_state(synch->owner, XNRELAX)) {
xnthread_set_info(sleeper, XNSWREP);
xnshadow_send_sig(sleeper, SIGDEBUG,
SIGDEBUG_MIGRATE_PRIOINV, 1);
} else
xnthread_clear_info(sleeper, XNSWREP);
}
struct xnthread *xnsynch_release(struct xnsynch *synch)
{
const int use_fastlock = xnsynch_fastlock_p(synch);
struct xnthread *newowner, *lastowner;
xnhandle_t lastownerh, newownerh;
struct xnpholder *holder;
spl_t s;
XENO_BUGON(NUCLEUS, !testbits(synch->status, XNSYNCH_OWNER));
lastownerh = xnthread_handle(xnpod_current_thread());
if (use_fastlock &&
likely(xnsynch_fast_release(xnsynch_fastlock(synch), lastownerh)))
return NULL;
xnlock_get_irqsave(&nklock, s);
trace_mark(xn_nucleus, synch_release, "synch %p", synch);
holder = getpq(&synch->pendq);
if (holder) {
newowner = link2thread(holder, plink);
newowner->wchan = NULL;
newowner->wwake = synch;
lastowner = synch->owner;
synch->owner = newowner;
xnthread_set_info(newowner, XNWAKEN);
xnpod_resume_thread(newowner, XNPEND);
if (testbits(synch->status, XNSYNCH_CLAIMED))
xnsynch_clear_boost(synch, lastowner);
newownerh = xnsynch_fast_set_claimed(xnthread_handle(newowner),
xnsynch_pended_p(synch));
} else {
newowner = NULL;
synch->owner = NULL;
newownerh = XN_NO_HANDLE;
}
if (use_fastlock) {
xnarch_atomic_t *lockp = xnsynch_fastlock(synch);
xnarch_atomic_set(lockp, newownerh);
}
xnlock_put_irqrestore(&nklock, s);
xnarch_post_graph_if(synch, 0, emptypq_p(&synch->pendq));
return newowner;
}
static void xnsynch_renice_thread(struct xnthread *thread,
struct xnthread *target)
{
/* Apply the scheduling policy of "target" to "thread" */
xnsched_track_policy(thread, target);
if (thread->wchan)
xnsynch_requeue_sleeper(thread);
#ifdef CONFIG_XENO_OPT_PERVASIVE
if (xnthread_test_state(thread, XNRELAX))
xnshadow_renice(thread);
else if (xnthread_test_state(thread, XNSHADOW))
xnthread_set_info(thread, XNPRIOSET);
#endif /* CONFIG_XENO_OPT_PERVASIVE */
}
/*
* Detect when a thread is about to relax while holding a
* synchronization object currently claimed by another thread, which
* bears the TWARNSW bit (thus advertising a concern about potential
* spurious relaxes and priority inversion). By relying on the claim
* queue, we restrict the checks to PIP-enabled objects, but that
* already covers most of the use cases anyway.
*/
void xnsynch_detect_claimed_relax(struct xnthread *owner)
{
struct xnpholder *hs, *ht;
struct xnthread *sleeper;
struct xnsynch *synch;
for (hs = getheadpq(&owner->claimq); hs != NULL;
hs = nextpq(&owner->claimq, hs)) {
synch = link2synch(hs);
for (ht = getheadpq(&synch->pendq); ht != NULL;
ht = nextpq(&synch->pendq, ht)) {
sleeper = link2thread(ht, plink);
if (xnthread_test_state(sleeper, XNTRAPSW)) {
xnthread_set_info(sleeper, XNSWREP);
xnshadow_send_sig(sleeper, SIGDEBUG,
SIGDEBUG_MIGRATE_PRIOINV, 1);
}
}
}
}
/**
* @internal
* @fn void watchdog_handler(struct xntimer *timer)
* @brief Process watchdog ticks.
*
* This internal routine handles incoming watchdog ticks to detect
* software lockups. It kills any offending thread which is found to
* monopolize the CPU so as to starve the Linux kernel for too long.
*
* @coretags{coreirq-only, atomic-entry}
*/
static void watchdog_handler(struct xntimer *timer)
{
struct xnsched *sched = xnsched_current();
struct xnthread *curr = sched->curr;
if (likely(xnthread_test_state(curr, XNROOT))) {
xnsched_reset_watchdog(sched);
return;
}
if (likely(++sched->wdcount < wd_timeout_arg))
return;
trace_cobalt_watchdog_signal(curr);
if (xnthread_test_state(curr, XNUSER)) {
printk(XENO_WARNING "watchdog triggered on CPU #%d -- runaway thread "
"'%s' signaled\n", xnsched_cpu(sched), curr->name);
xnthread_call_mayday(curr, SIGDEBUG_WATCHDOG);
} else {
printk(XENO_WARNING "watchdog triggered on CPU #%d -- runaway thread "
"'%s' canceled\n", xnsched_cpu(sched), curr->name);
/*
* On behalf on an IRQ handler, xnthread_cancel()
* would go half way cancelling the preempted
* thread. Therefore we manually raise XNKICKED to
* cause the next call to xnthread_suspend() to return
* early in XNBREAK condition, and XNCANCELD so that
* @thread exits next time it invokes
* xnthread_test_cancel().
*/
xnthread_set_info(curr, XNKICKED|XNCANCELD);
}
xnsched_reset_watchdog(sched);
}
xnflags_t xnsynch_acquire(struct xnsynch *synch, xnticks_t timeout,
xntmode_t timeout_mode)
{
struct xnthread *thread = xnpod_current_thread(), *owner;
xnhandle_t threadh = xnthread_handle(thread), fastlock, old;
const int use_fastlock = xnsynch_fastlock_p(synch);
spl_t s;
XENO_BUGON(NUCLEUS, !testbits(synch->status, XNSYNCH_OWNER));
trace_mark(xn_nucleus, synch_acquire, "synch %p", synch);
redo:
if (use_fastlock) {
xnarch_atomic_t *lockp = xnsynch_fastlock(synch);
fastlock = xnarch_atomic_cmpxchg(lockp,
XN_NO_HANDLE, threadh);
if (likely(fastlock == XN_NO_HANDLE)) {
if (xnthread_test_state(thread, XNOTHER))
xnthread_inc_rescnt(thread);
xnthread_clear_info(thread,
XNRMID | XNTIMEO | XNBREAK);
return 0;
}
xnlock_get_irqsave(&nklock, s);
/* Set claimed bit.
In case it appears to be set already, re-read its state
under nklock so that we don't miss any change between the
lock-less read and here. But also try to avoid cmpxchg
where possible. Only if it appears not to be set, start
with cmpxchg directly. */
if (xnsynch_fast_is_claimed(fastlock)) {
old = xnarch_atomic_get(lockp);
goto test_no_owner;
}
do {
old = xnarch_atomic_cmpxchg(lockp, fastlock,
xnsynch_fast_set_claimed(fastlock, 1));
if (likely(old == fastlock))
break;
test_no_owner:
if (old == XN_NO_HANDLE) {
/* Owner called xnsynch_release
(on another cpu) */
xnlock_put_irqrestore(&nklock, s);
goto redo;
}
fastlock = old;
} while (!xnsynch_fast_is_claimed(fastlock));
owner = xnthread_lookup(xnsynch_fast_mask_claimed(fastlock));
if (!owner) {
/* The handle is broken, therefore pretend that the synch
object was deleted to signal an error. */
xnthread_set_info(thread, XNRMID);
goto unlock_and_exit;
}
xnsynch_set_owner(synch, owner);
} else {
xnlock_get_irqsave(&nklock, s);
owner = synch->owner;
if (!owner) {
synch->owner = thread;
if (xnthread_test_state(thread, XNOTHER))
xnthread_inc_rescnt(thread);
xnthread_clear_info(thread,
XNRMID | XNTIMEO | XNBREAK);
goto unlock_and_exit;
}
}
xnsynch_detect_relaxed_owner(synch, thread);
if (!testbits(synch->status, XNSYNCH_PRIO)) /* i.e. FIFO */
appendpq(&synch->pendq, &thread->plink);
else if (w_cprio(thread) > w_cprio(owner)) {
if (xnthread_test_info(owner, XNWAKEN) && owner->wwake == synch) {
/* Ownership is still pending, steal the resource. */
synch->owner = thread;
xnthread_clear_info(thread, XNRMID | XNTIMEO | XNBREAK);
xnthread_set_info(owner, XNROBBED);
goto grab_and_exit;
}
insertpqf(&synch->pendq, &thread->plink, w_cprio(thread));
if (testbits(synch->status, XNSYNCH_PIP)) {
if (!xnthread_test_state(owner, XNBOOST)) {
owner->bprio = owner->cprio;
xnthread_set_state(owner, XNBOOST);
}
if (testbits(synch->status, XNSYNCH_CLAIMED))
removepq(&owner->claimq, &synch->link);
else
__setbits(synch->status, XNSYNCH_CLAIMED);
insertpqf(&owner->claimq, &synch->link, w_cprio(thread));
xnsynch_renice_thread(owner, thread);
}
} else
insertpqf(&synch->pendq, &thread->plink, w_cprio(thread));
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
thread->wwake = NULL;
xnthread_clear_info(thread, XNWAKEN);
if (xnthread_test_info(thread, XNRMID | XNTIMEO | XNBREAK))
goto unlock_and_exit;
if (xnthread_test_info(thread, XNROBBED)) {
/* Somebody stole us the ownership while we were ready
to run, waiting for the CPU: we need to wait again
for the resource. */
if (timeout_mode != XN_RELATIVE || timeout == XN_INFINITE) {
xnlock_put_irqrestore(&nklock, s);
goto redo;
}
timeout = xntimer_get_timeout_stopped(&thread->rtimer);
if (timeout > 1) { /* Otherwise, it's too late. */
xnlock_put_irqrestore(&nklock, s);
goto redo;
}
xnthread_set_info(thread, XNTIMEO);
} else {
grab_and_exit:
if (xnthread_test_state(thread, XNOTHER))
xnthread_inc_rescnt(thread);
if (use_fastlock) {
xnarch_atomic_t *lockp = xnsynch_fastlock(synch);
/* We are the new owner, update the fastlock
accordingly. */
if (xnsynch_pended_p(synch))
threadh =
xnsynch_fast_set_claimed(threadh, 1);
xnarch_atomic_set(lockp, threadh);
}
}
unlock_and_exit:
xnlock_put_irqrestore(&nklock, s);
return xnthread_test_info(thread, XNRMID|XNTIMEO|XNBREAK);
}
void xnsynch_sleep_on(xnsynch_t *synch, xnticks_t timeout,
xntmode_t timeout_mode)
{
xnthread_t *thread = xnpod_current_thread(), *owner;
spl_t s;
xnlock_get_irqsave(&nklock, s);
trace_mark(xn_nucleus_synch_sleepon,
"thread %p thread_name %s synch %p",
thread, xnthread_name(thread), synch);
if (!testbits(synch->status, XNSYNCH_PRIO)) { /* i.e. FIFO */
appendpq(&synch->pendq, &thread->plink);
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
goto unlock_and_exit;
}
if (!testbits(synch->status, XNSYNCH_PIP)) { /* i.e. no ownership */
insertpqf(&synch->pendq, &thread->plink, thread->cprio);
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
goto unlock_and_exit;
}
redo:
owner = synch->owner;
if (!owner) {
synch->owner = thread;
xnthread_clear_info(thread, XNRMID | XNTIMEO | XNBREAK);
goto unlock_and_exit;
}
if (thread->cprio > owner->cprio) {
if (xnthread_test_info(owner, XNWAKEN) && owner->wwake == synch) {
/* Ownership is still pending, steal the resource. */
synch->owner = thread;
xnthread_clear_info(thread, XNRMID | XNTIMEO | XNBREAK);
xnthread_set_info(owner, XNROBBED);
goto unlock_and_exit;
}
if (!xnthread_test_state(owner, XNBOOST)) {
owner->bprio = owner->cprio;
xnthread_set_state(owner, XNBOOST);
}
if (testbits(synch->status, XNSYNCH_CLAIMED))
removepq(&owner->claimq, &synch->link);
else
__setbits(synch->status, XNSYNCH_CLAIMED);
insertpqf(&owner->claimq, &synch->link, thread->cprio);
insertpqf(&synch->pendq, &thread->plink, thread->cprio);
xnsynch_renice_thread(owner, thread->cprio);
} else
insertpqf(&synch->pendq, &thread->plink, thread->cprio);
xnpod_suspend_thread(thread, XNPEND, timeout, timeout_mode, synch);
if (xnthread_test_info(thread, XNRMID | XNTIMEO | XNBREAK))
goto unlock_and_exit;
if (xnthread_test_info(thread, XNROBBED)) {
/* Somebody stole us the ownership while we were ready
to run, waiting for the CPU: we need to wait again
for the resource. */
if (timeout_mode != XN_RELATIVE || timeout == XN_INFINITE)
goto redo;
timeout = xntimer_get_timeout_stopped(&thread->rtimer);
if (timeout > 1) /* Otherwise, it's too late. */
goto redo;
xnthread_set_info(thread, XNTIMEO);
}
unlock_and_exit:
thread->wwake = NULL;
xnthread_clear_info(thread, XNWAKEN);
xnlock_put_irqrestore(&nklock, s);
}
static void xnthread_timeout_handler(xntimer_t *timer)
{
xnthread_t *thread = container_of(timer, xnthread_t, rtimer);
xnthread_set_info(thread, XNTIMEO); /* Interrupts are off. */
xnpod_resume_thread(thread, XNDELAY);
}