int init_fpu(struct task_struct *tsk) { if (tsk_used_math(tsk)) { if ((boot_cpu_data.flags & CPU_HAS_FPU) && tsk == current) unlazy_fpu(tsk, task_pt_regs(tsk)); return 0; } /* * Memory allocation at the first usage of the FPU and other state. */ if (!tsk->thread.xstate) { tsk->thread.xstate = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); if (!tsk->thread.xstate) return -ENOMEM; } if (boot_cpu_data.flags & CPU_HAS_FPU) { struct sh_fpu_hard_struct *fp = &tsk->thread.xstate->hardfpu; memset(fp, 0, xstate_size); fp->fpscr = FPSCR_INIT; } else { struct sh_fpu_soft_struct *fp = &tsk->thread.xstate->softfpu; memset(fp, 0, xstate_size); fp->fpscr = FPSCR_INIT; } set_stopped_child_used_math(tsk); return 0; }
static inline int save_sigcontext_fpu(struct sigcontext *sc) { struct task_struct *tsk = current; unsigned long flags; int val; if (!tsk->used_math) { val = 0; __copy_to_user(&sc->sc_ownedfp, &val, sizeof(int)); return 0; } val = 1; __copy_to_user(&sc->sc_ownedfp, &val, sizeof(int)); /* This will cause a "finit" to be triggered by the next attempted FPU operation by the 'current' process. */ tsk->used_math = 0; save_and_cli(flags); unlazy_fpu(tsk); restore_flags(flags); return __copy_to_user(&sc->sc_fpregs[0], &tsk->thread.fpu.hard, sizeof(long)*(16*2+2)); }
/*H:040 * This is the i386-specific code to setup and run the Guest. Interrupts * are disabled: we own the CPU. */ void lguest_arch_run_guest(struct lg_cpu *cpu) { /* * Remember the awfully-named TS bit? If the Guest has asked to set it * we set it now, so we can trap and pass that trap to the Guest if it * uses the FPU. */ if (cpu->ts) unlazy_fpu(current); /* * SYSENTER is an optimized way of doing system calls. We can't allow * it because it always jumps to privilege level 0. A normal Guest * won't try it because we don't advertise it in CPUID, but a malicious * Guest (or malicious Guest userspace program) could, so we tell the * CPU to disable it before running the Guest. */ if (boot_cpu_has(X86_FEATURE_SEP)) wrmsr(MSR_IA32_SYSENTER_CS, 0, 0); /* * Now we actually run the Guest. It will return when something * interesting happens, and we can examine its registers to see what it * was doing. */ run_guest_once(cpu, lguest_pages(raw_smp_processor_id())); /* * Note that the "regs" structure contains two extra entries which are * not really registers: a trap number which says what interrupt or * trap made the switcher code come back, and an error code which some * traps set. */ /* Restore SYSENTER if it's supposed to be on. */ if (boot_cpu_has(X86_FEATURE_SEP)) wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0); /* * If the Guest page faulted, then the cr2 register will tell us the * bad virtual address. We have to grab this now, because once we * re-enable interrupts an interrupt could fault and thus overwrite * cr2, or we could even move off to a different CPU. */ if (cpu->regs->trapnum == 14) cpu->arch.last_pagefault = read_cr2(); /* * Similarly, if we took a trap because the Guest used the FPU, * we have to restore the FPU it expects to see. * math_state_restore() may sleep and we may even move off to * a different CPU. So all the critical stuff should be done * before this. */ else if (cpu->regs->trapnum == 7) math_state_restore(); }
/* * fill in the FPU structure for a core dump */ int dump_fpu(struct pt_regs *regs, elf_fpregset_t *fpreg) { struct task_struct *tsk = current; int fpvalid; fpvalid = is_using_fpu(tsk); if (fpvalid) { unlazy_fpu(tsk); memcpy(fpreg, &tsk->thread.fpu_state, sizeof(*fpreg)); } return fpvalid; }
static inline int elf32_core_copy_task_xfpregs(struct task_struct *t, elf_fpxregset_t *xfpu) { struct pt_regs *regs = task_pt_regs(t); if (!tsk_used_math(t)) return 0; if (t == current) unlazy_fpu(t); memcpy(xfpu, &t->thread.i387.fxsave, sizeof(elf_fpxregset_t)); xfpu->fcs = regs->cs; xfpu->fos = t->thread.ds; /* right? */ return 1; }
void save_rest_processor_state(void) { if ( !is_idle_vcpu(current) ) unlazy_fpu(current); #if defined(CONFIG_X86_64) rdmsrl(MSR_CSTAR, saved_cstar); rdmsrl(MSR_LSTAR, saved_lstar); if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ) { rdmsrl(MSR_IA32_SYSENTER_ESP, saved_sysenter_esp); rdmsrl(MSR_IA32_SYSENTER_EIP, saved_sysenter_eip); } #endif }
/* * this gets called so that we can store lazy state into memory and copy the * current task into the new thread. */ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { int ret; unlazy_fpu(src); *dst = *src; if (fpu_allocated(&src->thread.fpu)) { memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu)); ret = fpu_alloc(&dst->thread.fpu); if (ret) return ret; fpu_copy(&dst->thread.fpu, &src->thread.fpu); } return 0; }
/* * this gets called so that we can store lazy state into memory and copy the * current task into the new thread. */ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { #ifdef CONFIG_SUPERH32 unlazy_fpu(src, task_pt_regs(src)); #endif *dst = *src; if (src->thread.xstate) { dst->thread.xstate = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); if (!dst->thread.xstate) return -ENOMEM; memcpy(dst->thread.xstate, src->thread.xstate, xstate_size); } return 0; }
/* * retrieve the contents of MN10300 userspace FPU registers */ static int fpuregs_get(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { const struct fpu_state_struct *fpregs = &target->thread.fpu_state; int ret; unlazy_fpu(target); ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fpregs, 0, sizeof(*fpregs)); if (ret < 0) return ret; return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, sizeof(*fpregs), -1); }
/* * Save the fpu, extended register state to the user signal frame. * * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save * state is copied. * 'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'. * * buf == buf_fx for 64-bit frames and 32-bit fsave frame. * buf != buf_fx for 32-bit frames with fxstate. * * If the fpu, extended register state is live, save the state directly * to the user frame pointed by the aligned pointer 'buf_fx'. Otherwise, * copy the thread's fpu state to the user frame starting at 'buf_fx'. * * If this is a 32-bit frame with fxstate, put a fsave header before * the aligned state at 'buf_fx'. * * For [f]xsave state, update the SW reserved fields in the [f]xsave frame * indicating the absence/presence of the extended state to the user. */ int save_xstate_sig(void __user *buf, void __user *buf_fx, int size) { struct xsave_struct *xsave = ¤t->thread.fpu.state->xsave; struct task_struct *tsk = current; int ia32_fxstate = (buf != buf_fx); ia32_fxstate &= (config_enabled(CONFIG_X86_32) || config_enabled(CONFIG_IA32_EMULATION)); if (!access_ok(VERIFY_WRITE, buf, size)) return -EACCES; if (!static_cpu_has(X86_FEATURE_FPU)) return fpregs_soft_get(current, NULL, 0, sizeof(struct user_i387_ia32_struct), NULL, (struct _fpstate_ia32 __user *) buf) ? -1 : 1; if (!config_enabled(CONFIG_L4) && user_has_fpu()) { /* Save the live register state to the user directly. */ if (save_user_xstate(buf_fx)) return -1; /* Update the thread's fxstate to save the fsave header. */ if (ia32_fxstate) fpu_fxsave(&tsk->thread.fpu); } else { if (config_enabled(CONFIG_L4) && user_has_fpu()) unlazy_fpu(tsk); sanitize_i387_state(tsk); if (__copy_to_user(buf_fx, xsave, xstate_size)) return -1; } /* Save the fsave header for the 32-bit frames. */ if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf)) return -1; if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate)) return -1; drop_init_fpu(tsk); /* trigger finit */ return 0; }
static inline int save_sigcontext_fpu(struct sigcontext *sc) { struct task_struct *tsk = current; if (!tsk->used_math) { __put_user(0, &sc->sc_ownedfp); return 0; } __put_user(1, &sc->sc_ownedfp); /* This will cause a "finit" to be triggered by the next attempted FPU operation by the 'current' process. */ tsk->used_math = 0; unlazy_fpu(tsk); return __copy_to_user(&sc->sc_fpregs[0], &tsk->thread.fpu.hard, sizeof(long)*(16*2+2)); }
int dump_fpu_ia32(struct pt_regs *regs, elf_fpregset_t *fp) { struct _fpstate_ia32 *fpu = (void*)fp; struct task_struct *tsk = current; mm_segment_t oldfs = get_fs(); int ret; if (!tsk->used_math) return 0; if (!(tsk->thread.flags & THREAD_IA32)) BUG(); unlazy_fpu(tsk); set_fs(KERNEL_DS); ret = save_i387_ia32(current, fpu, regs, 1); /* Correct for i386 bug. It puts the fop into the upper 16bits of the tag word (like FXSAVE), not into the fcs*/ fpu->cssel |= fpu->tag & 0xffff0000; set_fs(oldfs); return ret; }
void lguest_arch_run_guest(struct lg_cpu *cpu) { if (cpu->ts) unlazy_fpu(current); if (boot_cpu_has(X86_FEATURE_SEP)) wrmsr(MSR_IA32_SYSENTER_CS, 0, 0); run_guest_once(cpu, lguest_pages(raw_smp_processor_id())); if (boot_cpu_has(X86_FEATURE_SEP)) wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0); if (cpu->regs->trapnum == 14) cpu->arch.last_pagefault = read_cr2(); else if (cpu->regs->trapnum == 7) math_state_restore(); }
static inline int elf32_core_copy_task_fpregs(struct task_struct *tsk, struct pt_regs *regs, elf_fpregset_t *fpu) { struct _fpstate_ia32 *fpstate = (void*)fpu; mm_segment_t oldfs = get_fs(); if (!tsk_used_math(tsk)) return 0; if (!regs) regs = task_pt_regs(tsk); if (tsk == current) unlazy_fpu(tsk); set_fs(KERNEL_DS); save_i387_ia32(tsk, fpstate, regs, 1); /* Correct for i386 bug. It puts the fop into the upper 16bits of the tag word (like FXSAVE), not into the fcs*/ fpstate->cssel |= fpstate->tag & 0xffff0000; set_fs(oldfs); return 1; }
static inline int save_sigcontext_fpu(struct sigcontext __user *sc, struct pt_regs *regs) { struct task_struct *tsk = current; if (!(boot_cpu_data.flags & CPU_HAS_FPU)) return 0; if (!used_math()) return __put_user(0, &sc->sc_ownedfp); if (__put_user(1, &sc->sc_ownedfp)) return -EFAULT; /* This will cause a "finit" to be triggered by the next attempted FPU operation by the 'current' process. */ clear_used_math(); unlazy_fpu(tsk, regs); return __copy_to_user(&sc->sc_fpregs[0], &tsk->thread.xstate->hardfpu, sizeof(long)*(16*2+2)); }
static inline int setup_sigcontext_fpu(struct pt_regs *regs, struct sigcontext __user *sc) { struct task_struct *tsk = current; int ret = 0; __put_user_error(used_math(), &sc->used_math_flag, ret); if (!used_math()) return ret; preempt_disable(); #if IS_ENABLED(CONFIG_LAZY_FPU) if (last_task_used_math == tsk) save_fpu(last_task_used_math); #else unlazy_fpu(tsk); #endif ret = __copy_to_user(&sc->fpu, &tsk->thread.fpu, sizeof(struct fpu_struct)); preempt_enable(); return ret; }
/* * handle an FPU operational exception * - there's a possibility that if the FPU is asynchronous, the signal might * be meant for a process other than the current one */ asmlinkage void fpu_exception(struct pt_regs *regs, enum exception_code code) { struct task_struct *tsk = current; siginfo_t info; u32 fpcr; if (!user_mode(regs)) die_if_no_fixup("An FPU Operation exception happened in" " kernel space\n", regs, code); if (!is_using_fpu(tsk)) die_if_no_fixup("An FPU Operation exception happened," " but the FPU is not in use", regs, code); info.si_signo = SIGFPE; info.si_errno = 0; info.si_addr = (void *) tsk->thread.uregs->pc; info.si_code = FPE_FLTINV; unlazy_fpu(tsk); fpcr = tsk->thread.fpu_state.fpcr; if (fpcr & FPCR_EC_Z) info.si_code = FPE_FLTDIV; else if (fpcr & FPCR_EC_O) info.si_code = FPE_FLTOVF; else if (fpcr & FPCR_EC_U) info.si_code = FPE_FLTUND; else if (fpcr & FPCR_EC_I) info.si_code = FPE_FLTRES; force_sig_info(SIGFPE, &info, tsk); }
void prepare_to_export(struct task_struct *task) { if (!task->exit_state) unlazy_fpu(task); }
/* * This gets called before we allocate a new thread and copy * the current task into it. */ void prepare_to_copy(struct task_struct *tsk) { unlazy_fpu(tsk); }
/* * this gets called so that we can store lazy state into memory and copy the * current task into the new thread. */ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { unlazy_fpu(src); *dst = *src; return 0; }