asmlinkage void __sched __ipipe_preempt_schedule_irq(void)
{
struct ipipe_percpu_domain_data *p;
unsigned long flags;
BUG_ON(!hard_irqs_disabled());
local_irq_save(flags);
hard_local_irq_enable();
preempt_schedule_irq(); /* Ok, may reschedule now. */
hard_local_irq_disable();
/*
* Flush any pending interrupt that may have been logged after
* preempt_schedule_irq() stalled the root stage before
* returning to us, and now.
*/
p = ipipe_this_cpu_root_context();
if (unlikely(__ipipe_ipending_p(p))) {
add_preempt_count(PREEMPT_ACTIVE);
trace_hardirqs_on();
__clear_bit(IPIPE_STALL_FLAG, &p->status);
__ipipe_sync_stage();
sub_preempt_count(PREEMPT_ACTIVE);
}
__ipipe_restore_root_nosync(flags);
}
void enabled_wait(void)
{
struct s390_idle_data *idle = this_cpu_ptr(&s390_idle);
unsigned long long idle_time;
unsigned long psw_mask;
trace_hardirqs_on();
/* Wait for external, I/O or machine check interrupt. */
psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_DAT |
PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
clear_cpu_flag(CIF_NOHZ_DELAY);
/* Call the assembler magic in entry.S */
psw_idle(idle, psw_mask);
trace_hardirqs_off();
/* Account time spent with enabled wait psw loaded as idle time. */
write_seqcount_begin(&idle->seqcount);
idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
idle->idle_time += idle_time;
idle->idle_count++;
account_idle_time(idle_time);
write_seqcount_end(&idle->seqcount);
}
/*
* __ipipe_do_sync_stage() -- Flush the pending IRQs for the current
* domain (and processor). This routine flushes the interrupt log (see
* "Optimistic interrupt protection" from D. Stodolsky et al. for more
* on the deferred interrupt scheme). Every interrupt that occurred
* while the pipeline was stalled gets played.
*
* WARNING: CPU migration may occur over this routine.
*/
void __ipipe_do_sync_stage(void)
{
struct ipipe_percpu_domain_data *p;
struct ipipe_domain *ipd;
int irq;
p = __ipipe_current_context;
ipd = p->domain;
__set_bit(IPIPE_STALL_FLAG, &p->status);
smp_wmb();
if (ipd == ipipe_root_domain)
trace_hardirqs_off();
for (;;) {
irq = __ipipe_next_irq(p);
if (irq < 0)
break;
/*
* Make sure the compiler does not reorder wrongly, so
* that all updates to maps are done before the
* handler gets called.
*/
barrier();
if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
continue;
if (ipd != ipipe_head_domain)
hard_local_irq_enable();
if (likely(ipd != ipipe_root_domain)) {
ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
__ipipe_run_irqtail(irq);
hard_local_irq_disable();
} else if (ipipe_virtual_irq_p(irq)) {
irq_enter();
ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
irq_exit();
root_stall_after_handler();
hard_local_irq_disable();
while (__ipipe_check_root_resched())
__ipipe_preempt_schedule_irq();
} else {
ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
root_stall_after_handler();
hard_local_irq_disable();
}
p = __ipipe_current_context;
}
if (ipd == ipipe_root_domain)
trace_hardirqs_on();
__clear_bit(IPIPE_STALL_FLAG, &p->status);
}
void __ipipe_restore_root_nosync(unsigned long x)
{
struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context();
if (raw_irqs_disabled_flags(x)) {
__set_bit(IPIPE_STALL_FLAG, &p->status);
trace_hardirqs_off();
} else {
trace_hardirqs_on();
__clear_bit(IPIPE_STALL_FLAG, &p->status);
}
}
/*
* do_IRQ()
* Primary interface for handling IRQ() requests.
*
* This function is force executed when the PIC switches a thread to it.
*/
asmlinkage void do_IRQ(struct pt_regs *regs, unsigned int irq)
{
struct pt_regs *oldregs;
struct thread_info *ti = current_thread_info();
STOPWATCH_DECLARE;
trace_hardirqs_off();
/*
* Mark that we are inside of an interrupt and
* that interrupts are disabled.
*/
oldregs = set_irq_regs(regs);
ti->interrupt_nesting++;
irq_kernel_stack_check(irq, regs);
/*
* Start the interrupt sequence
*/
irq_enter();
/*
* Execute the IRQ handler and any pending SoftIRQ requests.
*/
BUG_ON(!irqs_disabled());
STOPWATCH_START();
generic_handle_irq(irq);
STOPWATCH_END(&irq_watches[irq]);
BUG_ON(!irqs_disabled());
/*
* Exit the interrupt and process softirqs if needed.
*/
STOPWATCH_START();
irq_exit();
STOPWATCH_END(&irq_watches[INTERRUPT_COUNT]);
BUG_ON(!irqs_disabled());
/*
* Outside of an interrupt (or nested exit).
*/
trace_hardirqs_on();
ti->interrupt_nesting--;
set_irq_regs(oldregs);
}
void __ipipe_halt_root(void)
{
struct ipipe_percpu_domain_data *p;
/* Emulate sti+hlt sequence over the root domain. */
local_irq_disable_hw();
p = ipipe_root_cpudom_ptr();
trace_hardirqs_on();
__clear_bit(IPIPE_STALL_FLAG, &p->status);
if (unlikely(__ipipe_ipending_p(p))) {
__ipipe_sync_pipeline();
local_irq_enable_hw();
} else {
#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
ipipe_trace_end(0x8000000E);
#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
asm volatile("sti; hlt": : :"memory");
}
}
void __irq_entry do_IRQ(struct pt_regs *regs)
{
unsigned int irq;
struct pt_regs *old_regs = set_irq_regs(regs);
trace_hardirqs_off();
irq_enter();
irq = xintc_get_irq();
next_irq:
BUG_ON(!irq);
generic_handle_irq(irq);
irq = xintc_get_irq();
if (irq != -1U) {
pr_debug("next irq: %d\n", irq);
++concurrent_irq;
goto next_irq;
}
irq_exit();
set_irq_regs(old_regs);
trace_hardirqs_on();
}
/*
* Common checks before entering the guest world. Call with interrupts
* disabled.
*
* returns:
*
* == 1 if we're ready to go into guest state
* <= 0 if we need to go back to the host with return value
*/
int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu)
{
int r = 1;
WARN_ON_ONCE(!irqs_disabled());
while (true) {
if (need_resched()) {
local_irq_enable();
cond_resched();
local_irq_disable();
continue;
}
if (signal_pending(current)) {
kvmppc_account_exit(vcpu, SIGNAL_EXITS);
vcpu->run->exit_reason = KVM_EXIT_INTR;
r = -EINTR;
break;
}
vcpu->mode = IN_GUEST_MODE;
/*
* Reading vcpu->requests must happen after setting vcpu->mode,
* so we don't miss a request because the requester sees
* OUTSIDE_GUEST_MODE and assumes we'll be checking requests
* before next entering the guest (and thus doesn't IPI).
*/
smp_mb();
if (vcpu->requests) {
/* Make sure we process requests preemptable */
local_irq_enable();
trace_kvm_check_requests(vcpu);
r = kvmppc_core_check_requests(vcpu);
local_irq_disable();
if (r > 0)
continue;
break;
}
if (kvmppc_core_prepare_to_enter(vcpu)) {
/* interrupts got enabled in between, so we
are back at square 1 */
continue;
}
#ifdef CONFIG_PPC64
/* lazy EE magic */
hard_irq_disable();
if (lazy_irq_pending()) {
/* Got an interrupt in between, try again */
local_irq_enable();
local_irq_disable();
kvm_guest_exit();
continue;
}
trace_hardirqs_on();
#endif
kvm_guest_enter();
break;
}
return r;
}
