static inline int
save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu)
{
int err = 0;
#ifdef CONFIG_SMP
if (test_tsk_thread_flag(current, TIF_USEDFPU)) {
put_psr(get_psr() | PSR_EF);
fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr,
¤t->thread.fpqueue[0], ¤t->thread.fpqdepth);
regs->psr &= ~(PSR_EF);
clear_tsk_thread_flag(current, TIF_USEDFPU);
}
#else
if (current == last_task_used_math) {
put_psr(get_psr() | PSR_EF);
fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr,
¤t->thread.fpqueue[0], ¤t->thread.fpqdepth);
last_task_used_math = NULL;
regs->psr &= ~(PSR_EF);
}
#endif
err |= __copy_to_user(&fpu->si_float_regs[0],
¤t->thread.float_regs[0],
(sizeof(unsigned long) * 32));
err |= __put_user(current->thread.fsr, &fpu->si_fsr);
err |= __put_user(current->thread.fpqdepth, &fpu->si_fpqdepth);
if (current->thread.fpqdepth != 0)
err |= __copy_to_user(&fpu->si_fpqueue[0],
¤t->thread.fpqueue[0],
((sizeof(unsigned long) +
(sizeof(unsigned long *)))*16));
clear_used_math();
return err;
}
static int
restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, unsigned long *prax)
{
unsigned int err = 0;
/* Always make any pending restarted system calls return -EINTR */
current_thread_info()->restart_block.fn = do_no_restart_syscall;
#define COPY(x) err |= __get_user(regs->x, &sc->x)
COPY(rdi); COPY(rsi); COPY(rbp); COPY(rsp); COPY(rbx);
COPY(rdx); COPY(rcx); COPY(rip);
COPY(r8);
COPY(r9);
COPY(r10);
COPY(r11);
COPY(r12);
COPY(r13);
COPY(r14);
COPY(r15);
/* Kernel saves and restores only the CS segment register on signals,
* which is the bare minimum needed to allow mixed 32/64-bit code.
* App's signal handler can save/restore other segments if needed. */
{
unsigned cs;
err |= __get_user(cs, &sc->cs);
regs->cs = cs | 3; /* Force into user mode */
}
{
unsigned int tmpflags;
err |= __get_user(tmpflags, &sc->eflags);
regs->eflags = (regs->eflags & ~0x40DD5) | (tmpflags & 0x40DD5);
regs->orig_rax = -1; /* disable syscall checks */
}
{
struct _fpstate __user * buf;
err |= __get_user(buf, &sc->fpstate);
if (buf) {
if (!access_ok(VERIFY_READ, buf, sizeof(*buf)))
goto badframe;
err |= restore_i387(buf);
} else {
struct task_struct *me = current;
if (used_math()) {
clear_fpu(me);
clear_used_math();
}
}
}
err |= __get_user(*prax, &sc->rax);
return err;
badframe:
return 1;
}
static inline int save_i387(struct _fpstate __user *buf)
{
struct task_struct *tsk = current;
int err = 0;
BUILD_BUG_ON(sizeof(struct user_i387_struct) !=
sizeof(tsk->thread.xstate->fxsave));
if ((unsigned long)buf % 16)
printk("save_i387: bad fpstate %p\n", buf);
if (!used_math())
return 0;
clear_used_math(); /* trigger finit */
if (task_thread_info(tsk)->status & TS_USEDFPU) {
err = save_i387_checking((struct i387_fxsave_struct __user *)
buf);
if (err)
return err;
task_thread_info(tsk)->status &= ~TS_USEDFPU;
stts();
} else {
if (__copy_to_user(buf, &tsk->thread.xstate->fxsave,
sizeof(struct i387_fxsave_struct)))
return -1;
}
return 1;
}
/*
* This restores directly out of user space. Exceptions are handled.
*/
int restore_i387_xstate(void __user *buf)
{
struct task_struct *tsk = current;
int err = 0;
if (!buf) {
if (used_math())
goto clear;
return 0;
} else if (!access_ok(VERIFY_READ, buf, sig_xstate_size))
return -EACCES;
if (!used_math()) {
err = init_fpu(tsk);
if (err)
return err;
}
user_fpu_begin();
if (use_xsave())
err = restore_user_xstate(buf);
else
err = fxrstor_checking((__force struct i387_fxsave_struct *)
buf);
if (unlikely(err)) {
/*
* Encountered an error while doing the restore from the
* user buffer, clear the fpu state.
*/
clear:
clear_fpu(tsk);
clear_used_math();
}
return err;
}
static inline int restore_sigcontext_fpu(struct pt_regs *regs,
struct sigcontext __user *sc)
{
struct task_struct *tsk = current;
unsigned long used_math_flag;
int ret = 0;
clear_used_math();
__get_user_error(used_math_flag, &sc->used_math_flag, ret);
if (!used_math_flag)
return 0;
set_used_math();
#if IS_ENABLED(CONFIG_LAZY_FPU)
preempt_disable();
if (current == last_task_used_math) {
last_task_used_math = NULL;
disable_ptreg_fpu(regs);
}
preempt_enable();
#else
clear_fpu(regs);
#endif
return __copy_from_user(&tsk->thread.fpu, &sc->fpu,
sizeof(struct fpu_struct));
}
void __init cpu_init (void)
{
int cpu_id = smp_processor_id();
if (test_and_set_bit(cpu_id, &cpu_initialized)) {
printk(KERN_WARNING "CPU#%d already initialized!\n", cpu_id);
for ( ; ; )
local_irq_enable();
}
printk(KERN_INFO "Initializing CPU#%d\n", cpu_id);
/* Set up and load the per-CPU TSS and LDT */
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
if (current->mm)
BUG();
/* Force FPU initialization */
current_thread_info()->status = 0;
clear_used_math();
#ifdef CONFIG_MMU
/* Set up MMU */
init_mmu();
#endif
/* Set up ICUIMASK */
outl(0x00070000, M32R_ICU_IMASK_PORTL); /* imask=111 */
}
/*
* This restores directly out of user space. Exceptions are handled.
*/
static inline int restore_i387(struct _fpstate __user *buf)
{
struct task_struct *tsk = current;
int err;
if (!used_math()) {
err = init_fpu(tsk);
if (err)
return err;
}
if (!(task_thread_info(current)->status & TS_USEDFPU)) {
clts();
task_thread_info(current)->status |= TS_USEDFPU;
}
err = restore_fpu_checking((__force struct i387_fxsave_struct *)buf);
if (unlikely(err)) {
/*
* Encountered an error while doing the restore from the
* user buffer, clear the fpu state.
*/
clear_fpu(tsk);
clear_used_math();
}
return err;
}
static inline int
setup_sigcontext_fpu(struct pt_regs *regs, struct sigcontext __user *sc)
{
int err = 0;
int fpvalid;
fpvalid = !!used_math();
err |= __put_user(fpvalid, &sc->sc_fpvalid);
if (! fpvalid)
return err;
if (current == last_task_used_math) {
enable_fpu();
save_fpu(current);
disable_fpu();
last_task_used_math = NULL;
regs->sr |= SR_FD;
}
err |= __copy_to_user(&sc->sc_fpregs[0], ¤t->thread.xstate->hardfpu,
(sizeof(long long) * 32) + (sizeof(int) * 1));
clear_used_math();
return err;
}
void flush_thread(void)
{
struct thread_info *thread = current_thread_info();
struct task_struct *tsk = current;
memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
memset(&thread->fpstate, 0, sizeof(union fp_state));
clear_used_math();
}
void flush_thread(void)
{
struct task_struct *tsk = current;
memset(tsk->thread.debugreg, 0, sizeof(unsigned long)*8);
memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
clear_tsk_thread_flag(tsk, TIF_DEBUG);
/*
* Forget coprocessor state..
*/
clear_fpu(tsk);
clear_used_math();
}
static int
restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, unsigned long *prax)
{
unsigned int err = 0;
/* Always make any pending restarted system calls return -EINTR */
current_thread_info()->restart_block.fn = do_no_restart_syscall;
#define COPY(x) err |= __get_user(regs->x, &sc->x)
COPY(rdi); COPY(rsi); COPY(rbp); COPY(rsp); COPY(rbx);
COPY(rdx); COPY(rcx); COPY(rip);
COPY(r8);
COPY(r9);
COPY(r10);
COPY(r11);
COPY(r12);
COPY(r13);
COPY(r14);
COPY(r15);
{
unsigned int tmpflags;
err |= __get_user(tmpflags, &sc->eflags);
regs->eflags = (regs->eflags & ~0x40DD5) | (tmpflags & 0x40DD5);
regs->orig_rax = -1; /* disable syscall checks */
}
{
struct _fpstate __user * buf;
err |= __get_user(buf, &sc->fpstate);
if (buf) {
if (!access_ok(VERIFY_READ, buf, sizeof(*buf)))
goto badframe;
err |= restore_i387(buf);
} else {
struct task_struct *me = current;
if (used_math()) {
clear_fpu(me);
clear_used_math();
}
}
}
err |= __get_user(*prax, &sc->rax);
return err;
badframe:
return 1;
}
void flush_thread(void)
{
struct task_struct *tsk = current;
#if defined(CONFIG_X86_32) && !defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_PAX_MEMORY_UDEREF)
loadsegment(gs, 0);
#endif
flush_ptrace_hw_breakpoint(tsk);
memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
/*
* Forget coprocessor state..
*/
tsk->fpu_counter = 0;
clear_fpu(tsk);
clear_used_math();
}
void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp)
{
unsigned long status;
/* New thread loses kernel privileges. */
status = regs->cp0_status & ~(ST0_CU0|ST0_CU1|ST0_FR|KU_MASK);
status |= KU_USER;
regs->cp0_status = status;
clear_used_math();
clear_fpu_owner();
init_dsp();
clear_thread_flag(TIF_USEDMSA);
clear_thread_flag(TIF_MSA_CTX_LIVE);
disable_msa();
regs->cp0_epc = pc;
regs->regs[29] = sp;
}
/*
* cpu_init() initializes state that is per-CPU.
*/
void __cpuinit cpu_init(void)
{
/*
* Store processor id in lowcore (used e.g. in timer_interrupt)
*/
get_cpu_id(&S390_lowcore.cpu_id);
/*
* Force FPU initialization:
*/
clear_thread_flag(TIF_USEDFPU);
clear_used_math();
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
BUG_ON(current->mm);
enter_lazy_tlb(&init_mm, current);
}
static void flush_thread_from_task(struct task_struct *tsk)
{
clear_tsk_thread_flag(tsk, TIF_DEBUG);
tsk->thread.debugreg0 = 0;
tsk->thread.debugreg1 = 0;
tsk->thread.debugreg2 = 0;
tsk->thread.debugreg3 = 0;
tsk->thread.debugreg6 = 0;
tsk->thread.debugreg7 = 0;
memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
/*
* Forget coprocessor state..
*/
tsk->fpu_counter = 0;
clear_fpu(tsk);
clear_used_math();
}
void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp)
{
unsigned long status;
/* New thread loses kernel privileges. */
status = regs->cp0_status & ~(ST0_CU0|ST0_CU1|ST0_FR|KU_MASK);
#ifdef CONFIG_64BIT
status |= test_thread_flag(TIF_32BIT_REGS) ? 0 : ST0_FR;
#endif
status |= KU_USER;
regs->cp0_status = status;
clear_used_math();
clear_fpu_owner();
if (cpu_has_dsp)
__init_dsp();
regs->cp0_epc = pc;
regs->regs[29] = sp;
}
void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp)
{
unsigned long status;
/* New thread loses kernel privileges. */
status = regs->cp0_status & ~(ST0_CU0|ST0_CU1|KU_MASK);
#ifdef CONFIG_64BIT
status &= ~ST0_FR;
status |= (current->thread.mflags & MF_32BIT_REGS) ? 0 : ST0_FR;
#endif
status |= KU_USER;
regs->cp0_status = status;
clear_used_math();
lose_fpu();
if (cpu_has_dsp)
__init_dsp();
regs->cp0_epc = pc;
regs->regs[29] = sp;
current_thread_info()->addr_limit = USER_DS;
}
/*
* cpu_init() initializes state that is per-CPU.
*/
void __devinit cpu_init (void)
{
int addr = hard_smp_processor_id();
/*
* Store processor id in lowcore (used e.g. in timer_interrupt)
*/
get_cpu_id(&S390_lowcore.cpu_data.cpu_id);
S390_lowcore.cpu_data.cpu_addr = addr;
/*
* Force FPU initialization:
*/
clear_thread_flag(TIF_USEDFPU);
clear_used_math();
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
if (current->mm)
BUG();
enter_lazy_tlb(&init_mm, current);
}
static inline int save_sigcontext_fpu(struct sigcontext __user *sc,
struct pt_regs *regs)
{
struct task_struct *tsk = current;
if (!(boot_cpu_data.flags & CPU_HAS_FPU))
return 0;
if (!used_math())
return __put_user(0, &sc->sc_ownedfp);
if (__put_user(1, &sc->sc_ownedfp))
return -EFAULT;
/* This will cause a "finit" to be triggered by the next
attempted FPU operation by the 'current' process.
*/
clear_used_math();
unlazy_fpu(tsk, regs);
return __copy_to_user(&sc->sc_fpregs[0], &tsk->thread.xstate->hardfpu,
sizeof(long)*(16*2+2));
}
int ckpt_restore_fpu(ckpt_desc_t desc)
{
int ret;
int flag;
log_restore_fpu("restoring fpu ...");
if (ckpt_read(desc, &flag, sizeof(int)) != sizeof(int)) {
log_err("failed to get file");
return -EIO;
}
kernel_fpu_begin();
clear_used_math();
if (flag) {
if (!ckpt_get_i387(current)) {
init_fpu(current);
if (!ckpt_get_i387(current)) {
log_err("failed to get i387");
return -EFAULT;
}
}
if (ckpt_read(desc, ckpt_get_i387(current), xstate_size) != xstate_size) {
log_err("failed to get i387");
return -EFAULT;
}
ret = ckpt_check_fpu_state();
if (ret) {
log_err("failed to restore i387");
return ret;
}
set_used_math();
}
kernel_fpu_end();
log_restore_pos(desc);
return 0;
}
static int
restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, int *r0_p)
{
unsigned int err = 0;
#define COPY(x) err |= __get_user(regs->x, &sc->sc_##x)
COPY(regs[1]);
COPY(regs[2]); COPY(regs[3]);
COPY(regs[4]); COPY(regs[5]);
COPY(regs[6]); COPY(regs[7]);
COPY(regs[8]); COPY(regs[9]);
COPY(regs[10]); COPY(regs[11]);
COPY(regs[12]); COPY(regs[13]);
COPY(regs[14]); COPY(regs[15]);
COPY(gbr); COPY(mach);
COPY(macl); COPY(pr);
COPY(sr); COPY(pc);
#undef COPY
#ifdef CONFIG_SH_FPU
if (boot_cpu_data.flags & CPU_HAS_FPU) {
int owned_fp;
struct task_struct *tsk = current;
regs->sr |= SR_FD; /* Release FPU */
clear_fpu(tsk, regs);
clear_used_math();
__get_user (owned_fp, &sc->sc_ownedfp);
if (owned_fp)
err |= restore_sigcontext_fpu(sc);
}
#endif
regs->tra = -1; /* disable syscall checks */
err |= __get_user(*r0_p, &sc->sc_regs[0]);
return err;
}
int save_i387_xstate(void __user *buf)
{
struct task_struct *tsk = current;
int err = 0;
if (!access_ok(VERIFY_WRITE, buf, sig_xstate_size))
return -EACCES;
BUG_ON(sig_xstate_size < xstate_size);
if ((unsigned long)buf % 64)
printk("save_i387_xstate: bad fpstate %p\n", buf);
if (!used_math())
return 0;
if (user_has_fpu()) {
if (use_xsave())
err = xsave_user(buf);
else
err = fxsave_user(buf);
if (err)
return err;
user_fpu_end();
} else {
sanitize_i387_state(tsk);
if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave,
xstate_size))
return -1;
}
clear_used_math(); /* trigger finit */
if (use_xsave()) {
struct _fpstate __user *fx = buf;
struct _xstate __user *x = buf;
u64 xstate_bv;
err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved,
sizeof(struct _fpx_sw_bytes));
err |= __put_user(FP_XSTATE_MAGIC2,
(__u32 __user *) (buf + sig_xstate_size
- FP_XSTATE_MAGIC2_SIZE));
/*
* Read the xstate_bv which we copied (directly from the cpu or
* from the state in task struct) to the user buffers and
* set the FP/SSE bits.
*/
err |= __get_user(xstate_bv, &x->xstate_hdr.xstate_bv);
/*
* For legacy compatible, we always set FP/SSE bits in the bit
* vector while saving the state to the user context. This will
* enable us capturing any changes(during sigreturn) to
* the FP/SSE bits by the legacy applications which don't touch
* xstate_bv in the xsave header.
*
* xsave aware apps can change the xstate_bv in the xsave
* header as well as change any contents in the memory layout.
* xrestore as part of sigreturn will capture all the changes.
*/
xstate_bv |= XSTATE_FPSSE;
err |= __put_user(xstate_bv, &x->xstate_hdr.xstate_bv);
if (err)
return err;
}
return 1;
}
static int
restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, int *peax)
{
unsigned int err = 0;
/* Always make any pending restarted system calls return -EINTR */
current_thread_info()->restart_block.fn = do_no_restart_syscall;
#define COPY(x) err |= __get_user(regs->x, &sc->x)
#define COPY_SEG(seg) \
{ unsigned short tmp; \
err |= __get_user(tmp, &sc->seg); \
regs->x##seg = tmp; }
#define COPY_SEG_STRICT(seg) \
{ unsigned short tmp; \
err |= __get_user(tmp, &sc->seg); \
regs->x##seg = tmp|3; }
#define GET_SEG(seg) \
{ unsigned short tmp; \
err |= __get_user(tmp, &sc->seg); \
loadsegment(seg,tmp); }
#define FIX_EFLAGS (X86_EFLAGS_AC | X86_EFLAGS_RF | \
X86_EFLAGS_OF | X86_EFLAGS_DF | \
X86_EFLAGS_TF | X86_EFLAGS_SF | X86_EFLAGS_ZF | \
X86_EFLAGS_AF | X86_EFLAGS_PF | X86_EFLAGS_CF)
GET_SEG(gs);
COPY_SEG(fs);
COPY_SEG(es);
COPY_SEG(ds);
COPY(edi);
COPY(esi);
COPY(ebp);
COPY(esp);
COPY(ebx);
COPY(edx);
COPY(ecx);
COPY(eip);
COPY_SEG_STRICT(cs);
COPY_SEG_STRICT(ss);
{
unsigned int tmpflags;
err |= __get_user(tmpflags, &sc->eflags);
regs->eflags = (regs->eflags & ~FIX_EFLAGS) | (tmpflags & FIX_EFLAGS);
regs->orig_eax = -1; /* disable syscall checks */
}
{
struct _fpstate __user * buf;
err |= __get_user(buf, &sc->fpstate);
if (buf) {
if (!access_ok(VERIFY_READ, buf, sizeof(*buf)))
goto badframe;
err |= restore_i387(buf);
} else {
struct task_struct *me = current;
if (used_math()) {
clear_fpu(me);
clear_used_math();
}
}
}
err |= __get_user(*peax, &sc->eax);
return err;
badframe:
return 1;
}
int save_i387_xstate(void __user *buf)
{
struct task_struct *tsk = current;
int err = 0;
if (!access_ok(VERIFY_WRITE, buf, sig_xstate_size))
return -EACCES;
BUG_ON(sig_xstate_size < xstate_size);
if ((unsigned long)buf % 64)
printk("save_i387_xstate: bad fpstate %p\n", buf);
if (!used_math())
return 0;
if (user_has_fpu()) {
if (use_xsave())
err = xsave_user(buf);
else
err = fxsave_user(buf);
if (err)
return err;
user_fpu_end();
} else {
sanitize_i387_state(tsk);
if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave,
xstate_size))
return -1;
}
clear_used_math(); /* */
if (use_xsave()) {
struct _fpstate __user *fx = buf;
struct _xstate __user *x = buf;
u64 xstate_bv;
err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved,
sizeof(struct _fpx_sw_bytes));
err |= __put_user(FP_XSTATE_MAGIC2,
(__u32 __user *) (buf + sig_xstate_size
- FP_XSTATE_MAGIC2_SIZE));
/*
*/
err |= __get_user(xstate_bv, &x->xstate_hdr.xstate_bv);
/*
*/
xstate_bv |= XSTATE_FPSSE;
err |= __put_user(xstate_bv, &x->xstate_hdr.xstate_bv);
if (err)
return err;
}
return 1;
}