/* * This routine is called on return from interrupt if any of the * TIF_WORK_MASK flags are set in thread_info->flags. It is * entered with interrupts disabled so we don't miss an event * that modified the thread_info flags. If any flag is set, we * handle it and return, and the calling assembly code will * re-disable interrupts, reload the thread flags, and call back * if more flags need to be handled. * * We return whether we need to check the thread_info flags again * or not. Note that we don't clear TIF_SINGLESTEP here, so it's * important that it be tested last, and then claim that we don't * need to recheck the flags. */ int do_work_pending(struct pt_regs *regs, u32 thread_info_flags) { /* If we enter in kernel mode, do nothing and exit the caller loop. */ if (!user_mode(regs)) return 0; if (thread_info_flags & _TIF_NEED_RESCHED) { schedule(); return 1; } #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC() if (thread_info_flags & _TIF_ASYNC_TLB) { do_async_page_fault(regs); return 1; } #endif if (thread_info_flags & _TIF_SIGPENDING) { do_signal(regs); return 1; } if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); return 1; } if (thread_info_flags & _TIF_SINGLESTEP) { single_step_once(regs); return 0; } panic("work_pending: bad flags %#x\n", thread_info_flags); }
asmlinkage int do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) { do { if (likely(thread_flags & _TIF_NEED_RESCHED)) { schedule(); } else { if (unlikely(!user_mode(regs))) return 0; local_irq_enable(); if (thread_flags & _TIF_SIGPENDING) { int restart = do_signal(regs, syscall); if (unlikely(restart)) { /* * Restart without handlers. * Deal with it without leaving * the kernel space. */ return restart; } syscall = 0; } else if (thread_flags & _TIF_UPROBE) { uprobe_notify_resume(regs); } else { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } } local_irq_disable(); thread_flags = current_thread_info()->flags; } while (thread_flags & _TIF_WORK_MASK); return 0; }
asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned int thread_flags) { /* * The assembly code enters us with IRQs off, but it hasn't * informed the tracing code of that for efficiency reasons. * Update the trace code with the current status. */ trace_hardirqs_off(); do { if (thread_flags & _TIF_NEED_RESCHED) { schedule(); } else { local_irq_enable(); if (thread_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } if (thread_flags & _TIF_FOREIGN_FPSTATE) fpsimd_restore_current_state(); } local_irq_disable(); thread_flags = READ_ONCE(current_thread_info()->flags); } while (thread_flags & _TIF_WORK_MASK); }
asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall) { if (test_thread_flag(TIF_SIGPENDING)) do_signal(regs, in_syscall); if (test_and_clear_thread_flag(TIF_NOTIFY_RESUME)) tracehook_notify_resume(regs); }
void do_notify_resume(struct pt_regs *regs) { /* * ASM glue gaurantees that this is only called when returning to * user mode */ if (test_and_clear_thread_flag(TIF_NOTIFY_RESUME)) tracehook_notify_resume(regs); }
void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs, orig_i0); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags) { if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned int thread_flags, int syscall) { if (thread_flags & _TIF_SIGPENDING) do_signal(regs, syscall); if (thread_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs, orig_i0); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned int in_syscall, unsigned int thread_flags) { local_irq_enable(); if (thread_flags & _TIF_SIGPENDING) { do_signal(regs, in_syscall); } else { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
/* * notification of userspace execution resumption * - triggered by current->work.notify_resume */ asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags, int syscall) { /* deal with pending signal delivery */ if (thread_info_flags & (1 << TIF_SIGPENDING)) do_signal(regs, syscall); if (thread_info_flags & (1 << TIF_NOTIFY_RESUME)) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
/* * notification of userspace execution resumption */ asmlinkage void do_notify_resume(struct pt_regs *regs) { if (test_thread_flag(TIF_SIGPENDING) || test_thread_flag(TIF_RESTORE_SIGMASK)) do_signal(regs); if (test_thread_flag(TIF_NOTIFY_RESUME)) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
asmlinkage void do_notify_resume(struct pt_regs *regs) { if (current_thread_info()->flags & _TIF_SIGPENDING) do_signal(regs); if (current_thread_info()->flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned int save_r0, unsigned long thread_info_flags) { /* deal with pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs, save_r0); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { user_exit(); if (thread_info_flags & _TIF_UPROBE) uprobe_notify_resume(regs); if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs, orig_i0); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } user_enter(); }
/* * notification of userspace execution resumption * - triggered by the _TIF_WORK_MASK flags */ asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { /* Handle pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) { do_signal(regs); } if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
void do_notify_resume(int canrestart, struct pt_regs *regs, __u32 thread_info_flags) { if (thread_info_flags & _TIF_SIGPENDING) do_signal(canrestart,regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
/* Called with IRQs disabled. */ __visible void prepare_exit_to_usermode(struct pt_regs *regs) { if (WARN_ON(!irqs_disabled())) local_irq_disable(); /* * In order to return to user mode, we need to have IRQs off with * none of _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_USER_RETURN_NOTIFY, * _TIF_UPROBE, or _TIF_NEED_RESCHED set. Several of these flags * can be set at any time on preemptable kernels if we have IRQs on, * so we need to loop. Disabling preemption wouldn't help: doing the * work to clear some of the flags can sleep. */ while (true) { u32 cached_flags = READ_ONCE(pt_regs_to_thread_info(regs)->flags); if (!(cached_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY))) break; /* We have work to do. */ local_irq_enable(); if (cached_flags & _TIF_NEED_RESCHED) schedule(); if (cached_flags & _TIF_UPROBE) uprobe_notify_resume(regs); /* deal with pending signal delivery */ if (cached_flags & _TIF_SIGPENDING) do_signal(regs); if (cached_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } if (cached_flags & _TIF_USER_RETURN_NOTIFY) fire_user_return_notifiers(); /* Disable IRQs and retry */ local_irq_disable(); } user_enter(); }
/* * notification of userspace execution resumption * - triggered by the TIF_WORK_MASK flags */ asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags) { local_irq_enable(); preempt_check_resched(); /* deal with pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
void do_notify_resume(struct pt_regs *regs, struct switch_stack *sw, unsigned long thread_info_flags, unsigned long r0, unsigned long r19) { if (thread_info_flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) do_signal(regs, sw, r0, r19); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned int thread_flags) { if (thread_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } if (thread_flags & _TIF_FOREIGN_FPSTATE) fpsimd_restore_current_state(); }
/* * notification of userspace execution resumption * - triggered by current->work.notify_resume */ asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags, int syscall) { /* deal with pending signal delivery */ if (thread_info_flags & ((1 << TIF_SIGPENDING) | (1 << TIF_RESTORE_SIGMASK))) do_signal(regs, syscall); if (thread_info_flags & (1 << TIF_NOTIFY_RESUME)) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
/* * notification of userspace execution resumption * - triggered by current->work.notify_resume */ void do_notify_resume(struct pt_regs *regs, __u32 thread_info_flags) { /* Pending single-step? */ if (thread_info_flags & _TIF_SINGLESTEP) clear_thread_flag(TIF_SINGLESTEP); /* deal with pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
/* * notification of userspace execution resumption * - triggered by the _TIF_WORK_MASK flags */ asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { /* Handle pending signal delivery */ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) { do_signal(regs); } if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
asmlinkage void do_notify_resume(struct pt_regs *regs, struct thread_info *ti) { int syscall = 0; if ((sysreg_read(SR) & MODE_MASK) == MODE_SUPERVISOR) syscall = 1; if (ti->flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) do_signal(regs, ¤t->blocked, syscall); if (ti->flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); } }
/* * notification of userspace execution resumption * - triggered by the TIF_WORK_MASK flags */ asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags) { local_irq_enable(); /* deal with pending signal delivery */ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } }
/* * notification of userspace execution resumption * - triggered by the TIF_WORK_MASK flags */ asmlinkage void do_notify_resume(__u32 thread_info_flags) { /* pending single-step? */ if (thread_info_flags & _TIF_SINGLESTEP) clear_thread_flag(TIF_SINGLESTEP); /* deal with pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) do_signal(); /* deal with notification on about to resume userspace execution */ if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(__frame); } } /* end do_notify_resume() */
asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall) { /* * We want the common case to go fast, which * is why we may in certain cases get here from * kernel mode. Just return without doing anything * if so. */ if (kernel_mode(regs)) return; if (test_thread_flag(TIF_SIGPENDING)) do_signal(regs, in_syscall); if (test_and_clear_thread_flag(TIF_NOTIFY_RESUME)) tracehook_notify_resume(regs); }
static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags) { /* * In order to return to user mode, we need to have IRQs off with * none of EXIT_TO_USERMODE_LOOP_FLAGS set. Several of these flags * can be set at any time on preemptible kernels if we have IRQs on, * so we need to loop. Disabling preemption wouldn't help: doing the * work to clear some of the flags can sleep. */ while (true) { /* We have work to do. */ local_irq_enable(); if (cached_flags & _TIF_NEED_RESCHED) schedule(); if (cached_flags & _TIF_UPROBE) uprobe_notify_resume(regs); if (cached_flags & _TIF_PATCH_PENDING) klp_update_patch_state(current); /* deal with pending signal delivery */ if (cached_flags & _TIF_SIGPENDING) do_signal(regs); if (cached_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); rseq_handle_notify_resume(NULL, regs); } if (cached_flags & _TIF_USER_RETURN_NOTIFY) fire_user_return_notifiers(); /* Disable IRQs and retry */ local_irq_disable(); cached_flags = READ_ONCE(current_thread_info()->flags); if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS)) break; } }
/* * notification of userspace execution resumption * - triggered by current->work.notify_resume */ void do_notify_resume(struct pt_regs *regs, __u32 thread_info_flags) { /* Pending single-step? */ if (thread_info_flags & _TIF_SINGLESTEP) clear_thread_flag(TIF_SINGLESTEP); /* deal with pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); if (current->replacement_session_keyring) key_replace_session_keyring(); } clear_thread_flag(TIF_IRET); }