asmlinkage void __sched __ipipe_preempt_schedule_irq(void) { struct ipipe_percpu_domain_data *p; unsigned long flags; BUG_ON(!hard_irqs_disabled()); local_irq_save(flags); hard_local_irq_enable(); preempt_schedule_irq(); /* Ok, may reschedule now. */ hard_local_irq_disable(); /* * Flush any pending interrupt that may have been logged after * preempt_schedule_irq() stalled the root stage before * returning to us, and now. */ p = ipipe_this_cpu_root_context(); if (unlikely(__ipipe_ipending_p(p))) { add_preempt_count(PREEMPT_ACTIVE); trace_hardirqs_on(); __clear_bit(IPIPE_STALL_FLAG, &p->status); __ipipe_sync_stage(); sub_preempt_count(PREEMPT_ACTIVE); } __ipipe_restore_root_nosync(flags); }
void enabled_wait(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); unsigned long long idle_time; unsigned long psw_mask; trace_hardirqs_on(); /* Wait for external, I/O or machine check interrupt. */ psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_DAT | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; clear_cpu_flag(CIF_NOHZ_DELAY); /* Call the assembler magic in entry.S */ psw_idle(idle, psw_mask); trace_hardirqs_off(); /* Account time spent with enabled wait psw loaded as idle time. */ write_seqcount_begin(&idle->seqcount); idle_time = idle->clock_idle_exit - idle->clock_idle_enter; idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; idle->idle_time += idle_time; idle->idle_count++; account_idle_time(idle_time); write_seqcount_end(&idle->seqcount); }
/* * __ipipe_do_sync_stage() -- Flush the pending IRQs for the current * domain (and processor). This routine flushes the interrupt log (see * "Optimistic interrupt protection" from D. Stodolsky et al. for more * on the deferred interrupt scheme). Every interrupt that occurred * while the pipeline was stalled gets played. * * WARNING: CPU migration may occur over this routine. */ void __ipipe_do_sync_stage(void) { struct ipipe_percpu_domain_data *p; struct ipipe_domain *ipd; int irq; p = __ipipe_current_context; ipd = p->domain; __set_bit(IPIPE_STALL_FLAG, &p->status); smp_wmb(); if (ipd == ipipe_root_domain) trace_hardirqs_off(); for (;;) { irq = __ipipe_next_irq(p); if (irq < 0) break; /* * Make sure the compiler does not reorder wrongly, so * that all updates to maps are done before the * handler gets called. */ barrier(); if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) continue; if (ipd != ipipe_head_domain) hard_local_irq_enable(); if (likely(ipd != ipipe_root_domain)) { ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); __ipipe_run_irqtail(irq); hard_local_irq_disable(); } else if (ipipe_virtual_irq_p(irq)) { irq_enter(); ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); irq_exit(); root_stall_after_handler(); hard_local_irq_disable(); while (__ipipe_check_root_resched()) __ipipe_preempt_schedule_irq(); } else { ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); root_stall_after_handler(); hard_local_irq_disable(); } p = __ipipe_current_context; } if (ipd == ipipe_root_domain) trace_hardirqs_on(); __clear_bit(IPIPE_STALL_FLAG, &p->status); }
void __ipipe_restore_root_nosync(unsigned long x) { struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context(); if (raw_irqs_disabled_flags(x)) { __set_bit(IPIPE_STALL_FLAG, &p->status); trace_hardirqs_off(); } else { trace_hardirqs_on(); __clear_bit(IPIPE_STALL_FLAG, &p->status); } }
/* * do_IRQ() * Primary interface for handling IRQ() requests. * * This function is force executed when the PIC switches a thread to it. */ asmlinkage void do_IRQ(struct pt_regs *regs, unsigned int irq) { struct pt_regs *oldregs; struct thread_info *ti = current_thread_info(); STOPWATCH_DECLARE; trace_hardirqs_off(); /* * Mark that we are inside of an interrupt and * that interrupts are disabled. */ oldregs = set_irq_regs(regs); ti->interrupt_nesting++; irq_kernel_stack_check(irq, regs); /* * Start the interrupt sequence */ irq_enter(); /* * Execute the IRQ handler and any pending SoftIRQ requests. */ BUG_ON(!irqs_disabled()); STOPWATCH_START(); generic_handle_irq(irq); STOPWATCH_END(&irq_watches[irq]); BUG_ON(!irqs_disabled()); /* * Exit the interrupt and process softirqs if needed. */ STOPWATCH_START(); irq_exit(); STOPWATCH_END(&irq_watches[INTERRUPT_COUNT]); BUG_ON(!irqs_disabled()); /* * Outside of an interrupt (or nested exit). */ trace_hardirqs_on(); ti->interrupt_nesting--; set_irq_regs(oldregs); }
void __ipipe_halt_root(void) { struct ipipe_percpu_domain_data *p; /* Emulate sti+hlt sequence over the root domain. */ local_irq_disable_hw(); p = ipipe_root_cpudom_ptr(); trace_hardirqs_on(); __clear_bit(IPIPE_STALL_FLAG, &p->status); if (unlikely(__ipipe_ipending_p(p))) { __ipipe_sync_pipeline(); local_irq_enable_hw(); } else { #ifdef CONFIG_IPIPE_TRACE_IRQSOFF ipipe_trace_end(0x8000000E); #endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ asm volatile("sti; hlt": : :"memory"); } }
void __irq_entry do_IRQ(struct pt_regs *regs) { unsigned int irq; struct pt_regs *old_regs = set_irq_regs(regs); trace_hardirqs_off(); irq_enter(); irq = xintc_get_irq(); next_irq: BUG_ON(!irq); generic_handle_irq(irq); irq = xintc_get_irq(); if (irq != -1U) { pr_debug("next irq: %d\n", irq); ++concurrent_irq; goto next_irq; } irq_exit(); set_irq_regs(old_regs); trace_hardirqs_on(); }
/* * Common checks before entering the guest world. Call with interrupts * disabled. * * returns: * * == 1 if we're ready to go into guest state * <= 0 if we need to go back to the host with return value */ int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu) { int r = 1; WARN_ON_ONCE(!irqs_disabled()); while (true) { if (need_resched()) { local_irq_enable(); cond_resched(); local_irq_disable(); continue; } if (signal_pending(current)) { kvmppc_account_exit(vcpu, SIGNAL_EXITS); vcpu->run->exit_reason = KVM_EXIT_INTR; r = -EINTR; break; } vcpu->mode = IN_GUEST_MODE; /* * Reading vcpu->requests must happen after setting vcpu->mode, * so we don't miss a request because the requester sees * OUTSIDE_GUEST_MODE and assumes we'll be checking requests * before next entering the guest (and thus doesn't IPI). */ smp_mb(); if (vcpu->requests) { /* Make sure we process requests preemptable */ local_irq_enable(); trace_kvm_check_requests(vcpu); r = kvmppc_core_check_requests(vcpu); local_irq_disable(); if (r > 0) continue; break; } if (kvmppc_core_prepare_to_enter(vcpu)) { /* interrupts got enabled in between, so we are back at square 1 */ continue; } #ifdef CONFIG_PPC64 /* lazy EE magic */ hard_irq_disable(); if (lazy_irq_pending()) { /* Got an interrupt in between, try again */ local_irq_enable(); local_irq_disable(); kvm_guest_exit(); continue; } trace_hardirqs_on(); #endif kvm_guest_enter(); break; } return r; }