void xnintr_clock_handler(void)
{
xnstat_exectime_t *prev;
struct xnsched *sched;
unsigned cpu;
cpu = xnarch_current_cpu();
if (!cpumask_test_cpu(cpu, &xnarch_supported_cpus)) {
xnarch_relay_tick();
return;
}
sched = xnpod_sched_slot(cpu);
prev = xnstat_exectime_switch(sched,
&nkclock.stat[xnsched_cpu(sched)].account);
xnstat_counter_inc(&nkclock.stat[xnsched_cpu(sched)].hits);
trace_mark(xn_nucleus, irq_enter, "irq %u", XNARCH_TIMER_IRQ);
trace_mark(xn_nucleus, tbase_tick, "base %s", nktbase.name);
++sched->inesting;
__setbits(sched->lflags, XNINIRQ);
xnlock_get(&nklock);
xntimer_tick_aperiodic();
xnlock_put(&nklock);
xnstat_exectime_switch(sched, prev);
if (--sched->inesting == 0) {
__clrbits(sched->lflags, XNINIRQ);
xnpod_schedule();
sched = xnpod_current_sched();
}
/*
* If the clock interrupt preempted a real-time thread, any
* transition to the root thread has already triggered a host
* tick propagation from xnpod_schedule(), so at this point,
* we only need to propagate the host tick in case the
* interrupt preempted the root thread.
*/
if (testbits(sched->lflags, XNHTICK) &&
xnthread_test_state(sched->curr, XNROOT))
xnintr_host_tick(sched);
trace_mark(xn_nucleus, irq_exit, "irq %u", XNARCH_TIMER_IRQ);
}
/*
* Low-level interrupt handler dispatching non-shared ISRs -- Called with
* interrupts off.
*/
static void xnintr_irq_handler(unsigned irq, void *cookie)
{
struct xnsched *sched = xnpod_current_sched();
xnstat_exectime_t *prev;
struct xnintr *intr;
xnticks_t start;
int s;
prev = xnstat_exectime_get_current(sched);
start = xnstat_exectime_now();
trace_mark(xn_nucleus, irq_enter, "irq %u", irq);
++sched->inesting;
__setbits(sched->lflags, XNINIRQ);
xnlock_get(&xnirqs[irq].lock);
#ifdef CONFIG_SMP
/*
* In SMP case, we have to reload the cookie under the per-IRQ
* lock to avoid racing with xnintr_detach. However, we
* assume that no CPU migration will occur while running the
* interrupt service routine, so the scheduler pointer will
* remain valid throughout this function.
*/
intr = xnarch_get_irq_cookie(irq);
if (unlikely(!intr)) {
s = 0;
goto unlock_and_exit;
}
#else
/* cookie always valid, attach/detach happens with IRQs disabled */
intr = cookie;
#endif
s = intr->isr(intr);
if (unlikely(s == XN_ISR_NONE)) {
if (++intr->unhandled == XNINTR_MAX_UNHANDLED) {
xnlogerr("%s: IRQ%d not handled. Disabling IRQ "
"line.\n", __FUNCTION__, irq);
s |= XN_ISR_NOENABLE;
}
} else {
xnstat_counter_inc(&intr->stat[xnsched_cpu(sched)].hits);
xnstat_exectime_lazy_switch(sched,
&intr->stat[xnsched_cpu(sched)].account,
start);
intr->unhandled = 0;
}
#ifdef CONFIG_SMP
unlock_and_exit:
#endif
xnlock_put(&xnirqs[irq].lock);
if (s & XN_ISR_PROPAGATE)
xnarch_chain_irq(irq);
else if (!(s & XN_ISR_NOENABLE))
xnarch_end_irq(irq);
xnstat_exectime_switch(sched, prev);
if (--sched->inesting == 0) {
__clrbits(sched->lflags, XNINIRQ);
xnpod_schedule();
}
trace_mark(xn_nucleus, irq_exit, "irq %u", irq);
}
/*
* Low-level interrupt handler dispatching the user-defined ISRs for
* shared interrupts -- Called with interrupts off.
*/
static void xnintr_shirq_handler(unsigned irq, void *cookie)
{
struct xnsched *sched = xnpod_current_sched();
xnintr_irq_t *shirq = &xnirqs[irq];
xnstat_exectime_t *prev;
xnticks_t start;
xnintr_t *intr;
int s = 0, ret;
prev = xnstat_exectime_get_current(sched);
start = xnstat_exectime_now();
trace_mark(xn_nucleus, irq_enter, "irq %u", irq);
++sched->inesting;
__setbits(sched->status, XNINIRQ);
xnlock_get(&shirq->lock);
intr = shirq->handlers;
while (intr) {
/*
* NOTE: We assume that no CPU migration will occur
* while running the interrupt service routine.
*/
ret = intr->isr(intr);
s |= ret;
if (ret & XN_ISR_HANDLED) {
xnstat_counter_inc(
&intr->stat[xnsched_cpu(sched)].hits);
xnstat_exectime_lazy_switch(sched,
&intr->stat[xnsched_cpu(sched)].account,
start);
start = xnstat_exectime_now();
}
intr = intr->next;
}
xnlock_put(&shirq->lock);
if (unlikely(s == XN_ISR_NONE)) {
if (++shirq->unhandled == XNINTR_MAX_UNHANDLED) {
xnlogerr("%s: IRQ%d not handled. Disabling IRQ "
"line.\n", __FUNCTION__, irq);
s |= XN_ISR_NOENABLE;
}
} else
shirq->unhandled = 0;
if (s & XN_ISR_PROPAGATE)
xnarch_chain_irq(irq);
else if (!(s & XN_ISR_NOENABLE))
xnarch_end_irq(irq);
if (--sched->inesting == 0) {
__clrbits(sched->status, XNINIRQ);
xnpod_schedule();
}
trace_mark(xn_nucleus, irq_exit, "irq %u", irq);
xnstat_exectime_switch(sched, prev);
}
/*
* Low-level interrupt handler dispatching the user-defined ISRs for
* shared edge-triggered interrupts -- Called with interrupts off.
*/
static void xnintr_edge_shirq_handler(unsigned irq, void *cookie)
{
const int MAX_EDGEIRQ_COUNTER = 128;
struct xnsched *sched = xnpod_current_sched();
xnintr_irq_t *shirq = &xnirqs[irq];
int s = 0, counter = 0, ret, code;
struct xnintr *intr, *end = NULL;
xnstat_exectime_t *prev;
xnticks_t start;
prev = xnstat_exectime_get_current(sched);
start = xnstat_exectime_now();
trace_mark(xn_nucleus, irq_enter, "irq %u", irq);
++sched->inesting;
__setbits(sched->lflags, XNINIRQ);
xnlock_get(&shirq->lock);
intr = shirq->handlers;
while (intr != end) {
xnstat_exectime_switch(sched,
&intr->stat[xnsched_cpu(sched)].account);
/*
* NOTE: We assume that no CPU migration will occur
* while running the interrupt service routine.
*/
ret = intr->isr(intr);
code = ret & ~XN_ISR_BITMASK;
s |= ret;
if (code == XN_ISR_HANDLED) {
end = NULL;
xnstat_counter_inc(
&intr->stat[xnsched_cpu(sched)].hits);
xnstat_exectime_lazy_switch(sched,
&intr->stat[xnsched_cpu(sched)].account,
start);
start = xnstat_exectime_now();
} else if (end == NULL)
end = intr;
if (counter++ > MAX_EDGEIRQ_COUNTER)
break;
if (!(intr = intr->next))
intr = shirq->handlers;
}
xnlock_put(&shirq->lock);
if (counter > MAX_EDGEIRQ_COUNTER)
xnlogerr
("xnintr_edge_shirq_handler() : failed to get the IRQ%d line free.\n",
irq);
if (unlikely(s == XN_ISR_NONE)) {
if (++shirq->unhandled == XNINTR_MAX_UNHANDLED) {
xnlogerr("%s: IRQ%d not handled. Disabling IRQ "
"line.\n", __FUNCTION__, irq);
s |= XN_ISR_NOENABLE;
}
} else
shirq->unhandled = 0;
if (s & XN_ISR_PROPAGATE)
xnarch_chain_irq(irq);
else if (!(s & XN_ISR_NOENABLE))
xnarch_end_irq(irq);
xnstat_exectime_switch(sched, prev);
if (--sched->inesting == 0) {
__clrbits(sched->lflags, XNINIRQ);
xnpod_schedule();
}
trace_mark(xn_nucleus, irq_exit, "irq %u", irq);
}
int __xnsched_run(struct xnsched *sched)
{
struct xnthread *prev, *next, *curr;
int switched, shadow;
spl_t s;
if (xnarch_escalate())
return 0;
trace_cobalt_schedule(sched);
xnlock_get_irqsave(&nklock, s);
curr = sched->curr;
/*
* CAUTION: xnthread_host_task(curr) may be unsynced and even
* stale if curr = &rootcb, since the task logged by
* leave_root() may not still be the current one. Use
* "current" for disambiguating.
*/
xntrace_pid(current->pid, xnthread_current_priority(curr));
reschedule:
switched = 0;
if (!test_resched(sched))
goto out;
next = xnsched_pick_next(sched);
if (next == curr) {
if (unlikely(xnthread_test_state(next, XNROOT))) {
if (sched->lflags & XNHTICK)
xnintr_host_tick(sched);
if (sched->lflags & XNHDEFER)
xnclock_program_shot(&nkclock, sched);
}
goto out;
}
prev = curr;
trace_cobalt_switch_context(prev, next);
if (xnthread_test_state(next, XNROOT))
xnsched_reset_watchdog(sched);
sched->curr = next;
shadow = 1;
if (xnthread_test_state(prev, XNROOT)) {
leave_root(prev);
shadow = 0;
} else if (xnthread_test_state(next, XNROOT)) {
if (sched->lflags & XNHTICK)
xnintr_host_tick(sched);
if (sched->lflags & XNHDEFER)
xnclock_program_shot(&nkclock, sched);
enter_root(next);
}
xnstat_exectime_switch(sched, &next->stat.account);
xnstat_counter_inc(&next->stat.csw);
switch_context(sched, prev, next);
/*
* Test whether we transitioned from primary mode to secondary
* over a shadow thread, caused by a call to xnthread_relax().
* In such a case, we are running over the regular schedule()
* tail code, so we have to skip our tail code.
*/
if (shadow && ipipe_root_p)
goto shadow_epilogue;
switched = 1;
sched = xnsched_finish_unlocked_switch(sched);
/*
* Re-read the currently running thread, this is needed
* because of relaxed/hardened transitions.
*/
curr = sched->curr;
xnthread_switch_fpu(sched);
xntrace_pid(current->pid, xnthread_current_priority(curr));
out:
if (switched &&
xnsched_maybe_resched_after_unlocked_switch(sched))
goto reschedule;
if (curr->lock_count)
sched->lflags |= XNINLOCK;
xnlock_put_irqrestore(&nklock, s);
return switched;
shadow_epilogue:
__ipipe_complete_domain_migration();
XENO_BUG_ON(COBALT, xnthread_current() == NULL);
/*
* Interrupts must be disabled here (has to be done on entry
* of the Linux [__]switch_to function), but it is what
* callers expect, specifically the reschedule of an IRQ
* handler that hit before we call xnsched_run in
* xnthread_suspend() when relaxing a thread.
*/
XENO_BUG_ON(COBALT, !hard_irqs_disabled());
return 1;
}
