void kunmap_atomic(void *kvaddr, enum km_type type) { #ifdef CONFIG_DEBUG_HIGHMEM unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); if (vaddr < FIXADDR_START) { // FIXME dec_preempt_count(); preempt_check_resched(); return; } if (vaddr != __fix_to_virt(FIX_KMAP_BEGIN+idx)) BUG(); /* * force other mappings to Oops if they'll try to access * this pte without first remap it */ pte_clear(&init_mm, vaddr, kmap_pte-idx); __flush_tlb_one(vaddr); #endif dec_preempt_count(); preempt_check_resched(); }
static inline void _local_bh_enable_ip(unsigned long ip) { WARN_ON_ONCE(in_irq() || irqs_disabled()); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_disable(); #endif /* * Are softirqs going to be turned on now: */ if (softirq_count() == SOFTIRQ_DISABLE_OFFSET) trace_softirqs_on(ip); /* * Keep preemption disabled until we are done with * softirq processing: */ sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1); if (unlikely(!in_interrupt() && local_softirq_pending())) do_softirq(); dec_preempt_count(); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_enable(); #endif preempt_check_resched(); }
static void xen_restore_fl(unsigned long flags) { struct vcpu_info *vcpu; /* convert from IF type flag */ flags = !(flags & X86_EFLAGS_IF); /* There's a one instruction preempt window here. We need to make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable(); vcpu = this_cpu_read(xen_vcpu); vcpu->evtchn_upcall_mask = flags; preempt_enable_no_resched(); /* Doesn't matter if we get preempted here, because any pending event will get dealt with anyway. */ if (flags == 0) { preempt_check_resched(); barrier(); /* unmask then check (avoid races) */ if (unlikely(vcpu->evtchn_upcall_pending)) xen_force_evtchn_callback(); } }
void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) { WARN_ON_ONCE(in_irq() || irqs_disabled()); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_disable(); #endif /* * Are softirqs going to be turned on now: */ if (softirq_count() == SOFTIRQ_DISABLE_OFFSET) trace_softirqs_on(ip); /* * Keep preemption disabled until we are done with * softirq processing: */ preempt_count_sub(cnt - 1); if (unlikely(!in_interrupt() && local_softirq_pending())) { /* * Run softirq if any pending. And do it in its own stack * as we may be calling this deep in a task call stack already. */ do_softirq(); } preempt_count_dec(); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_enable(); #endif preempt_check_resched(); }
void local_bh_enable(void) { #ifdef CONFIG_TRACE_IRQFLAGS unsigned long flags; WARN_ON_ONCE(in_irq()); #endif WARN_ON_ONCE(irqs_disabled()); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_save(flags); #endif /* * Are softirqs going to be turned on now: */ if (softirq_count() == SOFTIRQ_OFFSET) trace_softirqs_on((unsigned long)__builtin_return_address(0)); /* * Keep preemption disabled until we are done with * softirq processing: */ sub_preempt_count(SOFTIRQ_OFFSET - 1); if (unlikely(!in_interrupt() && local_softirq_pending())) do_softirq(); dec_preempt_count(); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_restore(flags); #endif preempt_check_resched(); }
void local_bh_enable_ip(unsigned long ip) { WARN_ON_ONCE(in_irq() || irqs_disabled()); sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1); if (unlikely(!in_interrupt() && local_softirq_pending())) do_softirq(); dec_preempt_count(); preempt_check_resched(); }
/* * Note that 'init' is a special process: it doesn't get signals it doesn't * want to handle. Thus you cannot kill init even with a SIGKILL even by * mistake. * * Note that we go through the signals twice: once to check the signals that * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. */ int do_signal(struct pt_regs *regs, sigset_t *oldset) { siginfo_t info; int signr; struct k_sigaction ka; #ifdef CONFIG_PREEMPT_RT /* * Fully-preemptible kernel does not need interrupts disabled: */ local_irq_enable(); preempt_check_resched(); #endif /* * We want the common case to go fast, which * is why we may in certain cases get here from * kernel mode. Just return without doing anything * if so. */ if (!user_mode(regs)) return 1; if (current->flags & PF_FREEZE) { refrigerator(0); goto no_signal; } if (!oldset) oldset = ¤t->blocked; signr = get_signal_to_deliver(&info, &ka, regs, NULL); if (signr > 0) { /* Whee! Actually deliver the signal. */ handle_signal(signr, &ka, &info, oldset, regs); return 1; } no_signal: /* Did we come from a system call? */ if (regs->tra >= 0) { /* Restart the system call - no handlers present */ if (regs->regs[0] == -ERESTARTNOHAND || regs->regs[0] == -ERESTARTSYS || regs->regs[0] == -ERESTARTNOINTR || regs->regs[0] == -ERESTART_RESTARTBLOCK) { regs->pc -= 2; } } return 0; }
void kunmap_atomic(void *kvaddr, enum km_type type) { #ifdef CONFIG_DEBUG_HIGHMEM unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; unsigned long idx = type + KM_TYPE_NR*smp_processor_id(); if (vaddr < FIXADDR_START) { // FIXME dec_preempt_count(); preempt_check_resched(); return; } BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN+idx)); /* XXX Fix - Anton */ #if 0 __flush_cache_one(vaddr); #else flush_cache_all(); #endif /* * force other mappings to Oops if they'll try to access * this pte without first remap it */ pte_clear(kmap_pte-idx); /* XXX Fix - Anton */ #if 0 __flush_tlb_one(vaddr); #else flush_tlb_all(); #endif #endif dec_preempt_count(); preempt_check_resched(); }
void local_bh_enable(void) { WARN_ON(irqs_disabled()); /* * Keep preemption disabled until we are done with * softirq processing: */ preempt_count() -= SOFTIRQ_OFFSET - 1; if (unlikely(!in_interrupt() && local_softirq_pending())) do_softirq(); dec_preempt_count(); preempt_check_resched(); }
void kunmap_atomic(void *kvaddr, enum km_type type) { #if defined(CONFIG_DEBUG_HIGHMEM) || defined(CONFIG_XEN) unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); if (vaddr < FIXADDR_START) { // FIXME dec_preempt_count(); preempt_check_resched(); return; } #endif #if defined(CONFIG_DEBUG_HIGHMEM) if (vaddr != __fix_to_virt(FIX_KMAP_BEGIN+idx)) BUG(); /* * force other mappings to Oops if they'll try to access * this pte without first remap it */ pte_clear(kmap_pte-idx); __flush_tlb_one(vaddr); #elif defined(CONFIG_XEN) /* * We must ensure there are no dangling pagetable references when * returning memory to Xen (decrease_reservation). * XXX TODO: We could make this faster by only zapping when * kmap_flush_unused is called but that is trickier and more invasive. */ pte_clear(kmap_pte-idx); #endif dec_preempt_count(); preempt_check_resched(); }
/* * notification of userspace execution resumption * - triggered by the TIF_WORK_MASK flags */ asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags) { local_irq_enable(); preempt_check_resched(); /* deal with pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
static int do_signal_pending(sigset_t *oldset, struct pt_regs *regs) { siginfo_t info; int signr; struct k_sigaction ka; int ret; int is32 = is_32bit_task(); #ifdef CONFIG_PREEMPT_RT /* * Fully-preemptible kernel does not need interrupts disabled: */ local_irq_enable(); preempt_check_resched(); #endif if (current_thread_info()->local_flags & _TLF_RESTORE_SIGMASK) oldset = ¤t->saved_sigmask; else if (!oldset) oldset = ¤t->blocked; signr = get_signal_to_deliver(&info, &ka, regs, NULL); /* Is there any syscall restart business here ? */ check_syscall_restart(regs, &ka, signr > 0); if (signr <= 0) { struct thread_info *ti = current_thread_info(); /* No signal to deliver -- put the saved sigmask back */ if (ti->local_flags & _TLF_RESTORE_SIGMASK) { ti->local_flags &= ~_TLF_RESTORE_SIGMASK; sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); } return 0; /* no signals delivered */ } /* * Reenable the DABR before delivering the signal to * user space. The DABR will have been cleared if it * triggered inside the kernel. */ if (current->thread.dabr) { set_dabr(current->thread.dabr); #if defined(CONFIG_BOOKE) mtspr(SPRN_DBCR0, current->thread.dbcr0); #endif } if (is32) { if (ka.sa.sa_flags & SA_SIGINFO) ret = handle_rt_signal32(signr, &ka, &info, oldset, regs); else ret = handle_signal32(signr, &ka, &info, oldset, regs); } else { ret = handle_rt_signal64(signr, &ka, &info, oldset, regs); } if (ret) { spin_lock_irq(¤t->sighand->siglock); sigorsets(¤t->blocked, ¤t->blocked, &ka.sa.sa_mask); if (!(ka.sa.sa_flags & SA_NODEFER)) sigaddset(¤t->blocked, signr); recalc_sigpending(); spin_unlock_irq(¤t->sighand->siglock); /* * A signal was successfully delivered; the saved sigmask is in * its frame, and we can clear the TLF_RESTORE_SIGMASK flag. */ current_thread_info()->local_flags &= ~_TLF_RESTORE_SIGMASK; /* * Let tracing know that we've done the handler setup. */ tracehook_signal_handler(signr, &info, &ka, regs, test_thread_flag(TIF_SINGLESTEP)); } return ret; }
/* * Note that 'init' is a special process: it doesn't get signals it doesn't * want to handle. Thus you cannot kill init even with a SIGKILL even by * mistake. * * Note that we go through the signals twice: once to check the signals that * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. */ static int do_signal(sigset_t *oldset, struct pt_regs *regs, int syscall) { struct k_sigaction ka; siginfo_t info; int signr; #ifdef CONFIG_PREEMPT_RT /* * Fully-preemptible kernel does not need interrupts disabled: */ local_irq_enable(); preempt_check_resched(); #endif /* * We want the common case to go fast, which * is why we may in certain cases get here from * kernel mode. Just return without doing anything * if so. */ if (!user_mode(regs)) return 0; if (try_to_freeze()) goto no_signal; single_step_clear(current); signr = get_signal_to_deliver(&info, &ka, regs, NULL); if (signr > 0) { handle_signal(signr, &ka, &info, oldset, regs, syscall); single_step_set(current); return 1; } no_signal: /* * No signal to deliver to the process - restart the syscall. */ if (syscall) { if (regs->ARM_r0 == -ERESTART_RESTARTBLOCK) { if (thumb_mode(regs)) { regs->ARM_r7 = __NR_restart_syscall - __NR_SYSCALL_BASE; regs->ARM_pc -= 2; } else { #if defined(CONFIG_AEABI) && !defined(CONFIG_OABI_COMPAT) regs->ARM_r7 = __NR_restart_syscall; regs->ARM_pc -= 4; #else u32 __user *usp; u32 swival = __NR_restart_syscall; regs->ARM_sp -= 12; usp = (u32 __user *)regs->ARM_sp; /* * Either we supports OABI only, or we have * EABI with the OABI compat layer enabled. * In the later case we don't know if user * space is EABI or not, and if not we must * not clobber r7. Always using the OABI * syscall solves that issue and works for * all those cases. */ swival = swival - __NR_SYSCALL_BASE + __NR_OABI_SYSCALL_BASE; put_user(regs->ARM_pc, &usp[0]); /* swi __NR_restart_syscall */ put_user(0xef000000 | swival, &usp[1]); /* ldr pc, [sp], #12 */ put_user(0xe49df00c, &usp[2]); flush_icache_range((unsigned long)usp, (unsigned long)(usp + 3)); regs->ARM_pc = regs->ARM_sp + 4; #endif } } if (regs->ARM_r0 == -ERESTARTNOHAND || regs->ARM_r0 == -ERESTARTSYS || regs->ARM_r0 == -ERESTARTNOINTR) { restart_syscall(regs); } } single_step_set(current); return 0; }
/* * Note that 'init' is a special process: it doesn't get signals it doesn't * want to handle. Thus you cannot kill init even with a SIGKILL even by * mistake. * * Note that we go through the signals twice: once to check the signals that * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. */ static void do_signal(struct pt_regs *regs, int syscall) { unsigned int retval = 0, continue_addr = 0, restart_addr = 0; struct k_sigaction ka; siginfo_t info; int signr; /* * We want the common case to go fast, which * is why we may in certain cases get here from * kernel mode. Just return without doing anything * if so. */ if (!user_mode(regs)) return; local_irq_enable(); preempt_check_resched(); /* * If we were from a system call, check for system call restarting... */ if (syscall) { continue_addr = regs->ARM_pc; restart_addr = continue_addr - (thumb_mode(regs) ? 2 : 4); retval = regs->ARM_r0; /* * Prepare for system call restart. We do this here so that a * debugger will see the already changed PSW. */ switch (retval) { case -ERESTARTNOHAND: case -ERESTARTSYS: case -ERESTARTNOINTR: regs->ARM_r0 = regs->ARM_ORIG_r0; regs->ARM_pc = restart_addr; break; case -ERESTART_RESTARTBLOCK: regs->ARM_r0 = -EINTR; break; } } if (try_to_freeze()) goto no_signal; /* * Get the signal to deliver. When running under ptrace, at this * point the debugger may change all our registers ... */ signr = get_signal_to_deliver(&info, &ka, regs, NULL); if (signr > 0) { sigset_t *oldset; /* * Depending on the signal settings we may need to revert the * decision to restart the system call. But skip this if a * debugger has chosen to restart at a different PC. */ if (regs->ARM_pc == restart_addr) { if (retval == -ERESTARTNOHAND || (retval == -ERESTARTSYS && !(ka.sa.sa_flags & SA_RESTART))) { regs->ARM_r0 = -EINTR; regs->ARM_pc = continue_addr; } } if (test_thread_flag(TIF_RESTORE_SIGMASK)) oldset = ¤t->saved_sigmask; else oldset = ¤t->blocked; if (handle_signal(signr, &ka, &info, oldset, regs) == 0) { /* * A signal was successfully delivered; the saved * sigmask will have been stored in the signal frame, * and will be restored by sigreturn, so we can simply * clear the TIF_RESTORE_SIGMASK flag. */ if (test_thread_flag(TIF_RESTORE_SIGMASK)) clear_thread_flag(TIF_RESTORE_SIGMASK); } return; } no_signal: if (syscall) { /* * Handle restarting a different system call. As above, * if a debugger has chosen to restart at a different PC, * ignore the restart. */ if (retval == -ERESTART_RESTARTBLOCK && regs->ARM_pc == continue_addr) { if (thumb_mode(regs)) { regs->ARM_r7 = __NR_restart_syscall - __NR_SYSCALL_BASE; regs->ARM_pc -= 2; } else { #if defined(CONFIG_AEABI) && !defined(CONFIG_OABI_COMPAT) regs->ARM_r7 = __NR_restart_syscall; regs->ARM_pc -= 4; #else u32 __user *usp; regs->ARM_sp -= 4; usp = (u32 __user *)regs->ARM_sp; if (put_user(regs->ARM_pc, usp) == 0) { regs->ARM_pc = KERN_RESTART_CODE; } else { regs->ARM_sp += 4; force_sigsegv(0, current); } #endif } } /* If there's no signal to deliver, we just put the saved sigmask * back. */ if (test_thread_flag(TIF_RESTORE_SIGMASK)) { clear_thread_flag(TIF_RESTORE_SIGMASK); sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); } } }
void do_signal32(struct pt_regs *regs) { struct k_sigaction ka; sigset_t *oldset; siginfo_t info; int signr; #ifdef CONFIG_PREEMPT_RT local_irq_enable(); preempt_check_resched(); #endif /* * We want the common case to go fast, which is why we may in certain * cases get here from kernel mode. Just return without doing anything * if so. */ if (!user_mode(regs)) return; if (try_to_freeze()) goto no_signal; if (test_thread_flag(TIF_RESTORE_SIGMASK)) oldset = ¤t->saved_sigmask; else oldset = ¤t->blocked; signr = get_signal_to_deliver(&info, &ka, regs, NULL); if (signr > 0) { /* Whee! Actually deliver the signal. */ if (handle_signal(signr, &info, &ka, oldset, regs) == 0) { /* * A signal was successfully delivered; the saved * sigmask will have been stored in the signal frame, * and will be restored by sigreturn, so we can simply * clear the TIF_RESTORE_SIGMASK flag. */ if (test_thread_flag(TIF_RESTORE_SIGMASK)) clear_thread_flag(TIF_RESTORE_SIGMASK); } } no_signal: /* * Who's code doesn't conform to the restartable syscall convention * dies here!!! The li instruction, a single machine instruction, * must directly be followed by the syscall instruction. */ if (regs->regs[0]) { if (regs->regs[2] == ERESTARTNOHAND || regs->regs[2] == ERESTARTSYS || regs->regs[2] == ERESTARTNOINTR) { regs->regs[7] = regs->regs[26]; regs->cp0_epc -= 8; } if (regs->regs[2] == ERESTART_RESTARTBLOCK) { regs->regs[2] = __NR_O32_restart_syscall; regs->regs[7] = regs->regs[26]; regs->cp0_epc -= 4; } } /* * If there's no signal to deliver, we just put the saved sigmask * back */ if (test_thread_flag(TIF_RESTORE_SIGMASK)) { clear_thread_flag(TIF_RESTORE_SIGMASK); sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); } }
/* * Note that 'init' is a special process: it doesn't get signals it doesn't * want to handle. Thus you cannot kill init even with a SIGKILL even by * mistake. */ static void do_signal(struct pt_regs *regs) { struct k_sigaction ka; siginfo_t info; int signr; sigset_t *oldset; #ifdef CONFIG_PREEMPT_RT /* * Fully-preemptible kernel does not need interrupts disabled: */ local_irq_enable(); preempt_check_resched(); #endif /* * We want the common case to go fast, which * is why we may in certain cases get here from * kernel mode. Just return without doing anything * if so. */ if (!user_mode(regs)) return; if (test_thread_flag(TIF_RESTORE_SIGMASK)) oldset = ¤t->saved_sigmask; else oldset = ¤t->blocked; signr = get_signal_to_deliver(&info, &ka, regs, NULL); if (signr > 0) { /* Reenable any watchpoints before delivering the * signal to user space. The processor register will * have been cleared if the watchpoint triggered * inside the kernel. */ if (current->thread.debugreg7) set_debugreg(current->thread.debugreg7, 7); /* Whee! Actually deliver the signal. */ if (handle_signal(signr, &info, &ka, oldset, regs) == 0) { /* a signal was successfully delivered; the saved * sigmask will have been stored in the signal frame, * and will be restored by sigreturn, so we can simply * clear the TIF_RESTORE_SIGMASK flag */ clear_thread_flag(TIF_RESTORE_SIGMASK); } return; } /* Did we come from a system call? */ if ((long)regs->orig_rax >= 0) { /* Restart the system call - no handlers present */ long res = regs->rax; switch (res) { case -ERESTARTNOHAND: case -ERESTARTSYS: case -ERESTARTNOINTR: regs->rax = regs->orig_rax; regs->rip -= 2; break; case -ERESTART_RESTARTBLOCK: regs->rax = test_thread_flag(TIF_IA32) ? __NR_ia32_restart_syscall : __NR_restart_syscall; regs->rip -= 2; break; } } /* if there's no signal to deliver, we just put the saved sigmask back. */ if (test_thread_flag(TIF_RESTORE_SIGMASK)) { clear_thread_flag(TIF_RESTORE_SIGMASK); sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); } }