void __ipipe_dispatch_irq_fast(unsigned int irq) /* hw interrupts off */
{
struct ipipe_percpu_domain_data *p = ipipe_this_cpu_leading_context(), *old;
struct ipipe_domain *head = p->domain;
if (unlikely(test_bit(IPIPE_STALL_FLAG, &p->status))) {
__ipipe_set_irq_pending(head, irq);
return;
}
old = __ipipe_current_context;
/* Switch to the head domain. */
__ipipe_set_current_context(p);
p->irqall[irq]++;
__set_bit(IPIPE_STALL_FLAG, &p->status);
barrier();
if (likely(head != ipipe_root_domain)) {
head->irqs[irq].handler(irq, head->irqs[irq].cookie);
__ipipe_run_irqtail(irq);
} else {
if (ipipe_virtual_irq_p(irq)) {
irq_enter();
head->irqs[irq].handler(irq, head->irqs[irq].cookie);
irq_exit();
} else
head->irqs[irq].handler(irq, head->irqs[irq].cookie);
root_stall_after_handler();
}
hard_local_irq_disable();
__clear_bit(IPIPE_STALL_FLAG, &p->status);
if (__ipipe_current_context == p) {
__ipipe_set_current_context(old);
if (old == p) {
if (__ipipe_ipending_p(p))
__ipipe_sync_stage();
return;
}
}
/*
* We must be running over the root domain, synchronize
* the pipeline for high priority IRQs.
*/
__ipipe_do_sync_pipeline(head);
}
/*
* __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
* interrupt protection log is maintained here for each domain. Hw
* interrupts are off on entry.
*/
int __ipipe_handle_irq(struct pt_regs *regs)
{
struct ipipe_domain *this_domain, *next_domain;
int irq, vector = regs->orig_ax;
struct list_head *head, *pos;
struct pt_regs *tick_regs;
int m_ack;
if (vector < 0) {
irq = __get_cpu_var(vector_irq)[~vector];
BUG_ON(irq < 0);
m_ack = 0;
} else { /* This is a self-triggered one. */
irq = vector;
m_ack = 1;
}
this_domain = ipipe_current_domain;
if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))
head = &this_domain->p_link;
else {
head = __ipipe_pipeline.next;
next_domain = list_entry(head, struct ipipe_domain, p_link);
if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
if (!m_ack && next_domain->irqs[irq].acknowledge)
next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
__ipipe_dispatch_wired(next_domain, irq);
goto finalize_nosync;
}
}
/* Ack the interrupt. */
pos = head;
while (pos != &__ipipe_pipeline) {
next_domain = list_entry(pos, struct ipipe_domain, p_link);
if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
__ipipe_set_irq_pending(next_domain, irq);
if (!m_ack && next_domain->irqs[irq].acknowledge) {
next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
m_ack = 1;
}
}
if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
break;
pos = next_domain->p_link.next;
}
/*
* If the interrupt preempted the head domain, then do not
* even try to walk the pipeline, unless an interrupt is
* pending for it.
*/
if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
!__ipipe_ipending_p(ipipe_head_cpudom_ptr()))
goto finalize_nosync;
/*
* Now walk the pipeline, yielding control to the highest
* priority domain that has pending interrupt(s) or
* immediately to the current domain if the interrupt has been
* marked as 'sticky'. This search does not go beyond the
* current domain in the pipeline.
*/
__ipipe_walk_pipeline(head);
finalize_nosync:
/*
* Given our deferred dispatching model for regular IRQs, we
* only record CPU regs for the last timer interrupt, so that
* the timer handler charges CPU times properly. It is assumed
* that other interrupt handlers don't actually care for such
* information.
*/
if (irq == __ipipe_hrtimer_irq) {
tick_regs = &__raw_get_cpu_var(__ipipe_tick_regs);
tick_regs->flags = regs->flags;
tick_regs->cs = regs->cs;
tick_regs->ip = regs->ip;
tick_regs->bp = regs->bp;
#ifdef CONFIG_X86_64
tick_regs->ss = regs->ss;
tick_regs->sp = regs->sp;
#endif
if (!__ipipe_root_domain_p)
tick_regs->flags &= ~X86_EFLAGS_IF;
}
if (user_mode(regs) && (current->ipipe_flags & PF_EVTRET) != 0) {
current->ipipe_flags &= ~PF_EVTRET;
__ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
}
if (!__ipipe_root_domain_p ||
test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)))
return 0;
return 1;
}
/*
* __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
* interrupt protection log is maintained here for each domain. Hw
* interrupts are masked on entry.
*/
void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
{
struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
struct ipipe_domain *this_domain, *next_domain;
struct list_head *head, *pos;
int m_ack, s = -1;
/*
* Software-triggered IRQs do not need any ack. The contents
* of the register frame should only be used when processing
* the timer interrupt, but not for handling any other
* interrupt.
*/
m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR);
this_domain = __ipipe_current_domain;
if (unlikely(test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)))
head = &this_domain->p_link;
else {
head = __ipipe_pipeline.next;
next_domain = list_entry(head, struct ipipe_domain, p_link);
if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
if (!m_ack && next_domain->irqs[irq].acknowledge != NULL)
next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
__ipipe_dispatch_wired(next_domain, irq);
goto out;
}
}
/* Ack the interrupt. */
pos = head;
while (pos != &__ipipe_pipeline) {
next_domain = list_entry(pos, struct ipipe_domain, p_link);
if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
__ipipe_set_irq_pending(next_domain, irq);
if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) {
next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
m_ack = 1;
}
}
if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
break;
pos = next_domain->p_link.next;
}
/*
* Now walk the pipeline, yielding control to the highest
* priority domain that has pending interrupt(s) or
* immediately to the current domain if the interrupt has been
* marked as 'sticky'. This search does not go beyond the
* current domain in the pipeline. We also enforce the
* additional root stage lock (blackfin-specific).
*/
if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
/*
* If the interrupt preempted the head domain, then do not
* even try to walk the pipeline, unless an interrupt is
* pending for it.
*/
if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
ipipe_head_cpudom_var(irqpend_himask) == 0)
goto out;
__ipipe_walk_pipeline(head);
out:
if (!s)
__clear_bit(IPIPE_STALL_FLAG, &p->status);
}
void __ipipe_dispatch_irq(unsigned int irq, int flags) /* hw interrupts off */
{
struct ipipe_domain *ipd;
struct irq_desc *desc;
unsigned long control;
/*
* Survival kit when reading this code:
*
* - we have two main situations, leading to three cases for
* handling interrupts:
*
* a) the root domain is alone, no registered head domain
* => all interrupts are delivered via the fast dispatcher.
* b) a head domain is registered
* => head domain IRQs go through the fast dispatcher
* => root domain IRQs go through the interrupt log
*
* - when no head domain is registered, ipipe_head_domain ==
* ipipe_root_domain == &ipipe_root.
*
* - the caller tells us whether we should acknowledge this
* IRQ. Even virtual IRQs may require acknowledge on some
* platforms (e.g. arm/SMP).
*
* - the caller tells us whether we may try to run the IRQ log
* syncer. Typically, demuxed IRQs won't be synced
* immediately.
*
* - multiplex IRQs most likely have a valid acknowledge
* handler and we may not be called with IPIPE_IRQF_NOACK
* for them. The ack handler for the multiplex IRQ actually
* decodes the demuxed interrupts.
*/
#ifdef CONFIG_IPIPE_DEBUG
if (unlikely(irq >= IPIPE_NR_IRQS) ||
(irq < NR_IRQS && irq_to_desc(irq) == NULL)) {
printk(KERN_ERR "I-pipe: spurious interrupt %u\n", irq);
return;
}
#endif
/*
* CAUTION: on some archs, virtual IRQs may have acknowledge
* handlers. Multiplex IRQs should have one too.
*/
desc = irq >= NR_IRQS ? NULL : irq_to_desc(irq);
if (flags & IPIPE_IRQF_NOACK)
IPIPE_WARN_ONCE(desc && ipipe_chained_irq_p(irq));
else {
ipd = ipipe_head_domain;
control = ipd->irqs[irq].control;
if ((control & IPIPE_HANDLE_MASK) == 0)
ipd = ipipe_root_domain;
if (ipd->irqs[irq].ackfn)
ipd->irqs[irq].ackfn(irq, desc);
if (desc && ipipe_chained_irq_p(irq)) {
if ((flags & IPIPE_IRQF_NOSYNC) == 0)
/* Run demuxed IRQ handlers. */
goto sync;
return;
}
}
/*
* Sticky interrupts must be handled early and separately, so
* that we always process them on the current domain.
*/
ipd = __ipipe_current_domain;
control = ipd->irqs[irq].control;
if (control & IPIPE_STICKY_MASK)
goto log;
/*
* In case we have no registered head domain
* (i.e. ipipe_head_domain == &ipipe_root), we allow
* interrupts to go through the fast dispatcher, since we
* don't care for additional latency induced by interrupt
* disabling at CPU level. Otherwise, we must go through the
* interrupt log, and leave the dispatching work ultimately to
* __ipipe_sync_pipeline().
*/
ipd = ipipe_head_domain;
control = ipd->irqs[irq].control;
if (control & IPIPE_HANDLE_MASK) {
if (unlikely(flags & IPIPE_IRQF_NOSYNC))
__ipipe_set_irq_pending(ipd, irq);
else
__ipipe_dispatch_irq_fast(irq);
return;
}
/*
* The root domain must handle all interrupts, so testing the
* HANDLE bit for it would be pointless.
*/
ipd = ipipe_root_domain;
log:
__ipipe_set_irq_pending(ipd, irq);
if (flags & IPIPE_IRQF_NOSYNC)
return;
/*
* Optimize if we preempted a registered high priority head
* domain: we don't need to synchronize the pipeline unless
* there is a pending interrupt for it.
*/
if (!__ipipe_root_p &&
!__ipipe_ipending_p(ipipe_this_cpu_head_context()))
return;
sync:
__ipipe_sync_pipeline(ipipe_head_domain);
}
