static int default_idle(void)
{
long oldval;
unsigned int cpu = smp_processor_id();
while (1) {
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
while (!need_resched() && !cpu_is_offline(cpu)) {
barrier();
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
}
HMT_medium();
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
schedule();
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
}
return 0;
}
/*
* Notification of system call entry/exit
* - triggered by current->work.syscall_trace
*/
asmlinkage long syscall_trace_enter(struct pt_regs *regs, long syscall)
{
long ret = 0;
user_exit();
current_thread_info()->syscall = syscall;
if (secure_computing() == -1)
return -1;
if (test_thread_flag(TIF_SYSCALL_TRACE) &&
tracehook_report_syscall_entry(regs))
ret = -1;
#ifdef CONFIG_GRKERNSEC_SETXID
if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
gr_delayed_cred_worker();
#endif
if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
trace_sys_enter(regs, regs->regs[2]);
audit_syscall_entry(syscall, regs->regs[4], regs->regs[5],
regs->regs[6], regs->regs[7]);
return syscall;
}
asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall)
{
if (test_thread_flag(TIF_SIGPENDING))
do_signal(regs, in_syscall);
if (test_and_clear_thread_flag(TIF_NOTIFY_RESUME))
tracehook_notify_resume(regs);
}
void do_notify_resume(struct pt_regs *regs)
{
/*
* ASM glue gaurantees that this is only called when returning to
* user mode
*/
if (test_and_clear_thread_flag(TIF_NOTIFY_RESUME))
tracehook_notify_resume(regs);
}
static int pseries_dedicated_idle(void)
{
long oldval;
struct paca_struct *lpaca = get_paca();
unsigned int cpu = smp_processor_id();
unsigned long start_snooze;
unsigned long *smt_snooze_delay = &__get_cpu_var(smt_snooze_delay);
while (1) {
/*
* Indicate to the HV that we are idle. Now would be
* a good time to find other work to dispatch.
*/
lpaca->lppaca.idle = 1;
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
start_snooze = __get_tb() +
*smt_snooze_delay * tb_ticks_per_usec;
while (!need_resched() && !cpu_is_offline(cpu)) {
ppc64_runlatch_off();
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
if (*smt_snooze_delay != 0 &&
__get_tb() > start_snooze) {
HMT_medium();
dedicated_idle_sleep(cpu);
}
}
HMT_medium();
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
lpaca->lppaca.idle = 0;
ppc64_runlatch_on();
schedule();
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
}
}
static void enable_TSC(void)
{
preempt_disable();
if (test_and_clear_thread_flag(TIF_NOTSC))
/*
* Must flip the CPU state synchronously with
* TIF_NOTSC in the current running context.
*/
hard_enable_TSC();
preempt_enable();
}
/*
* Load an updated userland FPSIMD state for 'current' from memory and set the
* flag that indicates that the FPSIMD register contents are the most recent
* FPSIMD state of 'current'
*/
void fpsimd_update_current_state(struct fpsimd_state *state)
{
preempt_disable();
fpsimd_load_state(state);
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
struct fpsimd_state *st = ¤t->thread.fpsimd_state;
this_cpu_write(fpsimd_last_state, st);
st->cpu = smp_processor_id();
}
preempt_enable();
}
static void enable_cpuid(void)
{
preempt_disable();
if (test_and_clear_thread_flag(TIF_NOCPUID)) {
/*
* Must flip the CPU state synchronously with
* TIF_NOCPUID in the current running context.
*/
set_cpuid_faulting(false);
}
preempt_enable();
}
asmlinkage int syscall_trace(int why, struct pt_regs *regs, int scno)
{
unsigned long ip;
current_thread_info()->syscall = scno;
if (why)
audit_syscall_exit(regs);
else {
if (secure_computing(scno) == -1)
return -1;
audit_syscall_entry(AUDIT_ARCH_ARM, scno, regs->ARM_r0,
regs->ARM_r1, regs->ARM_r2, regs->ARM_r3);
}
if (why == 0 && test_and_clear_thread_flag(TIF_SYSCALL_RESTARTSYS))
scno = __NR_restart_syscall - __NR_SYSCALL_BASE;
if (!test_thread_flag(TIF_SYSCALL_TRACE))
return scno;
if (!(current->ptrace & PT_PTRACED))
return scno;
/*
* IP is used to denote syscall entry/exit:
* IP = 0 -> entry, =1 -> exit
*/
ip = regs->ARM_ip;
regs->ARM_ip = why;
/*
* IP is used to denote syscall entry/exit:
* IP = 0 -> entry, =1 -> exit
*/
ip = regs->ARM_ip;
regs->ARM_ip = why;
/* the 0x80 provides a way for the tracing parent to distinguish
between a syscall stop and SIGTRAP delivery */
ptrace_notify(SIGTRAP | ((current->ptrace & PT_TRACESYSGOOD)
? 0x80 : 0));
/*
* this isn't the same as continuing with a signal, but it will do
* for normal use. strace only continues with a signal if the
* stopping signal is not SIGTRAP. -brl
*/
if (current->exit_code) {
send_sig(current->exit_code, current, 1);
current->exit_code = 0;
}
regs->ARM_ip = ip;
return current_thread_info()->syscall;
}
/*
* Load the userland FPSIMD state of 'current' from memory, but only if the
* FPSIMD state already held in the registers is /not/ the most recent FPSIMD
* state of 'current'
*/
void fpsimd_restore_current_state(void)
{
if (!system_supports_fpsimd())
return;
preempt_disable();
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
struct fpsimd_state *st = ¤t->thread.fpsimd_state;
fpsimd_load_state(st);
this_cpu_write(fpsimd_last_state, st);
st->cpu = smp_processor_id();
}
preempt_enable();
}
asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall)
{
/*
* We want the common case to go fast, which
* is why we may in certain cases get here from
* kernel mode. Just return without doing anything
* if so.
*/
if (kernel_mode(regs))
return;
if (test_thread_flag(TIF_SIGPENDING))
do_signal(regs, in_syscall);
if (test_and_clear_thread_flag(TIF_NOTIFY_RESUME))
tracehook_notify_resume(regs);
}
static inline void poll_idle(void)
{
int oldval;
local_irq_enable();
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
while (!need_resched())
cpu_relax();
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
}
static int iSeries_idle(void)
{
struct paca_struct *lpaca;
long oldval;
unsigned long CTRL;
/* ensure iSeries run light will be out when idle */
clear_thread_flag(TIF_RUN_LIGHT);
CTRL = mfspr(CTRLF);
CTRL &= ~RUNLATCH;
mtspr(CTRLT, CTRL);
lpaca = get_paca();
while (1) {
if (lpaca->lppaca.xSharedProc) {
if (ItLpQueue_isLpIntPending(lpaca->lpqueue_ptr))
process_iSeries_events();
if (!need_resched())
yield_shared_processor();
} else {
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
while (!need_resched()) {
HMT_medium();
if (ItLpQueue_isLpIntPending(lpaca->lpqueue_ptr))
process_iSeries_events();
HMT_low();
}
HMT_medium();
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
}
schedule();
}
return 0;
}
/*
* On SMP it's slightly faster (but much more power-consuming!)
* to poll the ->work.need_resched flag instead of waiting for the
* cross-CPU IPI to arrive. Use this option with caution.
*/
static inline void poll_idle(void)
{
int oldval;
local_irq_enable();
/*
* Deal with another CPU just having chosen a thread to
* run here:
*/
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
while (!need_resched())
cpu_relax();
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
}
int dedicated_idle(void)
{
long oldval;
struct paca_struct *lpaca = get_paca(), *ppaca;
unsigned long start_snooze;
unsigned long *smt_snooze_delay = &__get_cpu_var(smt_snooze_delay);
unsigned int cpu = smp_processor_id();
ppaca = &paca[cpu ^ 1];
while (1) {
/*
* Indicate to the HV that we are idle. Now would be
* a good time to find other work to dispatch.
*/
lpaca->lppaca.xIdle = 1;
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
start_snooze = __get_tb() +
*smt_snooze_delay * tb_ticks_per_usec;
while (!need_resched() && !cpu_is_offline(cpu)) {
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
if (*smt_snooze_delay == 0 ||
__get_tb() < start_snooze)
continue;
HMT_medium();
if (!(ppaca->lppaca.xIdle)) {
local_irq_disable();
/*
* We are about to sleep the thread
* and so wont be polling any
* more.
*/
clear_thread_flag(TIF_POLLING_NRFLAG);
/*
* SMT dynamic mode. Cede will result
* in this thread going dormant, if the
* partner thread is still doing work.
* Thread wakes up if partner goes idle,
* an interrupt is presented, or a prod
* occurs. Returning from the cede
* enables external interrupts.
*/
if (!need_resched())
cede_processor();
else
local_irq_enable();
} else {
/*
* Give the HV an opportunity at the
* processor, since we are not doing
* any work.
*/
poll_pending();
}
}
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
HMT_medium();
lpaca->lppaca.xIdle = 0;
schedule();
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
}
return 0;
}
/*
* Note that 'init' is a special process: it doesn't get signals it doesn't
* want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake.
*
* Note that we go through the signals twice: once to check the signals that
* the kernel can handle, and then we build all the user-level signal handling
* stack-frames in one go after that.
*/
static void do_signal(struct pt_regs *regs, int syscall)
{
struct k_sigaction ka;
siginfo_t info;
int signr;
/*
* We want the common case to go fast, which
* is why we may in certain cases get here from
* kernel mode. Just return without doing anything
* if so.
*/
if (!user_mode(regs))
return;
if (try_to_freeze())
goto no_signal;
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
if (signr > 0) {
sigset_t *oldset;
if (test_thread_flag(TIF_RESTORE_SIGMASK))
oldset = ¤t->saved_sigmask;
else
oldset = ¤t->blocked;
if (handle_signal(signr, &ka, &info, oldset, regs, syscall)
== 0) {
/*
* A signal was successfully delivered; the saved
* sigmask will have been stored in the signal frame,
* and will be restored by sigreturn, so we can simply
* clear the TIF_RESTORE_SIGMASK flag.
*/
if (test_thread_flag(TIF_RESTORE_SIGMASK))
clear_thread_flag(TIF_RESTORE_SIGMASK);
}
return;
}
no_signal:
/*
* No signal to deliver to the process - restart the syscall.
*/
if (syscall) {
if (regs->UCreg_00 == -ERESTART_RESTARTBLOCK) {
u32 __user *usp;
regs->UCreg_sp -= 4;
usp = (u32 __user *)regs->UCreg_sp;
if (put_user(regs->UCreg_pc, usp) == 0) {
regs->UCreg_pc = KERN_RESTART_CODE;
} else {
regs->UCreg_sp += 4;
force_sigsegv(0, current);
}
}
if (regs->UCreg_00 == -ERESTARTNOHAND ||
regs->UCreg_00 == -ERESTARTSYS ||
regs->UCreg_00 == -ERESTARTNOINTR) {
setup_syscall_restart(regs);
}
}
/* If there's no signal to deliver, we just put the saved
* sigmask back.
*/
if (test_and_clear_thread_flag(TIF_RESTORE_SIGMASK))
set_current_blocked(¤t->saved_sigmask);
}
/*
* Note that 'init' is a special process: it doesn't get signals it doesn't
* want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake.
*
* Note that we go through the signals twice: once to check the signals that
* the kernel can handle, and then we build all the user-level signal handling
* stack-frames in one go after that.
*/
static void do_signal(struct pt_regs *regs, int syscall)
{
unsigned int retval = 0, continue_addr = 0, restart_addr = 0;
struct k_sigaction ka;
siginfo_t info;
int signr;
/*
* We want the common case to go fast, which
* is why we may in certain cases get here from
* kernel mode. Just return without doing anything
* if so.
*/
if (!user_mode(regs))
return;
/*
* If we were from a system call, check for system call restarting...
*/
if (syscall) {
continue_addr = regs->ARM_pc;
restart_addr = continue_addr - (thumb_mode(regs) ? 2 : 4);
retval = regs->ARM_r0;
/*
* Prepare for system call restart. We do this here so that a
* debugger will see the already changed PSW.
*/
switch (retval) {
case -ERESTARTNOHAND:
case -ERESTARTSYS:
case -ERESTARTNOINTR:
regs->ARM_r0 = regs->ARM_ORIG_r0;
regs->ARM_pc = restart_addr;
break;
case -ERESTART_RESTARTBLOCK:
regs->ARM_r0 = -EINTR;
break;
}
}
/*
* Get the signal to deliver. When running under ptrace, at this
* point the debugger may change all our registers ...
*/
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
if (signr > 0) {
sigset_t *oldset;
/*
* Depending on the signal settings we may need to revert the
* decision to restart the system call. But skip this if a
* debugger has chosen to restart at a different PC.
*/
if (regs->ARM_pc == restart_addr) {
if (retval == -ERESTARTNOHAND
|| (retval == -ERESTARTSYS
&& !(ka.sa.sa_flags & SA_RESTART))) {
regs->ARM_r0 = -EINTR;
regs->ARM_pc = continue_addr;
}
}
if (test_thread_flag(TIF_RESTORE_SIGMASK))
oldset = ¤t->saved_sigmask;
else
oldset = ¤t->blocked;
if (handle_signal(signr, &ka, &info, oldset, regs) == 0) {
/*
* A signal was successfully delivered; the saved
* sigmask will have been stored in the signal frame,
* and will be restored by sigreturn, so we can simply
* clear the TIF_RESTORE_SIGMASK flag.
*/
if (test_thread_flag(TIF_RESTORE_SIGMASK))
clear_thread_flag(TIF_RESTORE_SIGMASK);
}
return;
}
if (syscall) {
/*
* Handle restarting a different system call. As above,
* if a debugger has chosen to restart at a different PC,
* ignore the restart.
*/
if (retval == -ERESTART_RESTARTBLOCK
&& regs->ARM_pc == continue_addr) {
if (thumb_mode(regs)) {
regs->ARM_r7 = __NR_restart_syscall - __NR_SYSCALL_BASE;
regs->ARM_pc -= 2;
} else {
#if defined(CONFIG_AEABI) && !defined(CONFIG_OABI_COMPAT)
regs->ARM_r7 = __NR_restart_syscall;
regs->ARM_pc -= 4;
#else
u32 __user *usp;
regs->ARM_sp -= 4;
usp = (u32 __user *)regs->ARM_sp;
if (put_user(regs->ARM_pc, usp) == 0) {
regs->ARM_pc = KERN_RESTART_CODE;
} else {
regs->ARM_sp += 4;
force_sigsegv(0, current);
}
#endif
}
}
}
/* If there's no signal to deliver, we just put the saved sigmask
* back.
*/
if (test_and_clear_thread_flag(TIF_RESTORE_SIGMASK))
set_current_blocked(¤t->saved_sigmask);
}
static int
handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
sigset_t *oldset, struct pt_regs *regs)
{
int ret;
#ifdef DEBUG_SIG
printk("handle_signal pid:%d sig:%lu rip:%lx rsp:%lx regs=%p\n",
current->pid, sig,
regs->rip, regs->rsp, regs);
#endif
/* Are we from a system call? */
if (current_syscall(regs) >= 0) {
/* If so, check system call restarting.. */
switch (current_syscall_ret(regs)) {
case -ERESTART_RESTARTBLOCK:
case -ERESTARTNOHAND:
regs->rax = -EINTR;
break;
case -ERESTARTSYS:
if (!(ka->sa.sa_flags & SA_RESTART)) {
regs->rax = -EINTR;
break;
}
/* fallthrough */
case -ERESTARTNOINTR:
regs->rax = regs->orig_rax;
regs->rip -= 2;
break;
}
}
/*
* If TF is set due to a debugger (TIF_FORCED_TF), clear the TF flag so
* that register information in the sigcontext is correct.
*/
if (unlikely(regs->eflags & TF_MASK)
&& likely(test_and_clear_thread_flag(TIF_FORCED_TF)))
regs->eflags &= ~TF_MASK;
#ifdef CONFIG_IA32_EMULATION
if (test_thread_flag(TIF_IA32)) {
if (ka->sa.sa_flags & SA_SIGINFO)
ret = ia32_setup_rt_frame(sig, ka, info, oldset, regs);
else
ret = ia32_setup_frame(sig, ka, oldset, regs);
} else
#endif
ret = setup_rt_frame(sig, ka, info, oldset, regs);
if (ret == 0) {
spin_lock_irq(¤t->sighand->siglock);
sigorsets(¤t->blocked,¤t->blocked,&ka->sa.sa_mask);
if (!(ka->sa.sa_flags & SA_NODEFER))
sigaddset(¤t->blocked,sig);
recalc_sigpending();
spin_unlock_irq(¤t->sighand->siglock);
/*
* Clear TF when entering the signal handler, but
* notify any tracer that was single-stepping it.
* The tracer may want to single-step inside the
* handler too.
*/
regs->eflags &= ~(TF_MASK | X86_EFLAGS_DF);
tracehook_report_handle_signal(sig, ka, oldset, regs);
}
return ret;
}
/*
* OK, we're invoking a handler:
*/
static int
handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
sigset_t *oldset, struct pt_regs *regs)
{
int ret;
/* Are we from a system call? */
if ((long)regs->orig_ax >= 0) {
/* If so, check system call restarting.. */
switch (regs->ax) {
case -ERESTART_RESTARTBLOCK:
case -ERESTARTNOHAND:
regs->ax = -EINTR;
break;
case -ERESTARTSYS:
if (!(ka->sa.sa_flags & SA_RESTART)) {
regs->ax = -EINTR;
break;
}
/* fallthrough */
case -ERESTARTNOINTR:
regs->ax = regs->orig_ax;
regs->ip -= 2;
break;
}
}
/*
* If TF is set due to a debugger (TIF_FORCED_TF), clear the TF
* flag so that register information in the sigcontext is correct.
*/
if (unlikely(regs->flags & X86_EFLAGS_TF) &&
likely(test_and_clear_thread_flag(TIF_FORCED_TF)))
regs->flags &= ~X86_EFLAGS_TF;
/* Set up the stack frame */
if (ka->sa.sa_flags & SA_SIGINFO)
ret = setup_rt_frame(sig, ka, info, oldset, regs);
else
ret = setup_frame(sig, ka, oldset, regs);
if (ret)
return ret;
/*
* Clear the direction flag as per the ABI for function entry.
*/
regs->flags &= ~X86_EFLAGS_DF;
/*
* Clear TF when entering the signal handler, but
* notify any tracer that was single-stepping it.
* The tracer may want to single-step inside the
* handler too.
*/
regs->flags &= ~X86_EFLAGS_TF;
if (test_thread_flag(TIF_SINGLESTEP))
ptrace_notify(SIGTRAP);
spin_lock_irq(¤t->sighand->siglock);
sigorsets(¤t->blocked, ¤t->blocked, &ka->sa.sa_mask);
if (!(ka->sa.sa_flags & SA_NODEFER))
sigaddset(¤t->blocked, sig);
recalc_sigpending();
spin_unlock_irq(¤t->sighand->siglock);
return 0;
}
static int
handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
sigset_t *oldset, struct pt_regs *regs)
{
int ret;
/* Are we from a system call? */
if (syscall_get_nr(current, regs) >= 0) {
/* If so, check system call restarting.. */
switch (syscall_get_error(current, regs)) {
case -ERESTART_RESTARTBLOCK:
case -ERESTARTNOHAND:
regs->ax = -EINTR;
break;
case -ERESTARTSYS:
if (!(ka->sa.sa_flags & SA_RESTART)) {
regs->ax = -EINTR;
break;
}
/* fallthrough */
case -ERESTARTNOINTR:
regs->ax = regs->orig_ax;
regs->ip -= 2;
break;
}
}
/*
* If TF is set due to a debugger (TIF_FORCED_TF), clear the TF
* flag so that register information in the sigcontext is correct.
*/
if (unlikely(regs->flags & X86_EFLAGS_TF) &&
likely(test_and_clear_thread_flag(TIF_FORCED_TF)))
regs->flags &= ~X86_EFLAGS_TF;
ret = setup_rt_frame(sig, ka, info, oldset, regs);
if (ret)
return ret;
#ifdef CONFIG_X86_64
/*
* This has nothing to do with segment registers,
* despite the name. This magic affects uaccess.h
* macros' behavior. Reset it to the normal setting.
*/
set_fs(USER_DS);
#endif
/*
* Clear the direction flag as per the ABI for function entry.
*/
regs->flags &= ~X86_EFLAGS_DF;
/*
* Clear TF when entering the signal handler, but
* notify any tracer that was single-stepping it.
* The tracer may want to single-step inside the
* handler too.
*/
regs->flags &= ~X86_EFLAGS_TF;
spin_lock_irq(¤t->sighand->siglock);
sigorsets(¤t->blocked, ¤t->blocked, &ka->sa.sa_mask);
if (!(ka->sa.sa_flags & SA_NODEFER))
sigaddset(¤t->blocked, sig);
recalc_sigpending();
spin_unlock_irq(¤t->sighand->siglock);
tracehook_signal_handler(sig, info, ka, regs,
test_thread_flag(TIF_SINGLESTEP));
return 0;
}