void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
{
u32 fpscr, orig_fpscr, fpsid, exceptions;
pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
atomic64_inc(&vfp_bounce_count);
fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK));
fpsid = fmrx(FPSID);
orig_fpscr = fpscr = fmrx(FPSCR);
if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT)
&& (fpscr & FPSCR_IXE)) {
goto emulate;
}
if (fpexc & FPEXC_EX) {
#ifndef CONFIG_CPU_FEROCEON
trigger = fmrx(FPINST);
regs->ARM_pc -= 4;
#endif
} else if (!(fpexc & FPEXC_DEX)) {
vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs);
goto exit;
}
if (fpexc & (FPEXC_EX | FPEXC_VV)) {
u32 len;
len = fpexc + (1 << FPEXC_LENGTH_BIT);
fpscr &= ~FPSCR_LENGTH_MASK;
fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT);
}
exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
if (exceptions)
vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
if (fpexc ^ (FPEXC_EX | FPEXC_FP2V))
goto exit;
barrier();
trigger = fmrx(FPINST2);
emulate:
exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
if (exceptions)
vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
exit:
preempt_enable();
}
void vfp_sync_hwstate(struct thread_info *thread)
{
unsigned int cpu = get_cpu();
/*
* If the thread we're interested in is the current owner of the
* hardware VFP state, then we need to save its state.
*/
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
u32 fpexc = fmrx(FPEXC);
/*
* Save the last VFP state on this CPU.
*/
fmxr(FPEXC, fpexc | FPEXC_EN);
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
fmxr(FPEXC, fpexc);
}
put_cpu();
}
/*
* Process bitmask of exception conditions.
*/
static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs)
{
int si_code = 0;
pr_debug("VFP: raising exceptions %08x\n", exceptions);
if (exceptions == VFP_EXCEPTION_ERROR) {
vfp_panic("unhandled bounce", inst);
vfp_raise_sigfpe(0, regs);
return;
}
/*
* If any of the status flags are set, update the FPSCR.
* Comparison instructions always return at least one of
* these flags set.
*/
/*
ARM spec does not define that H/W have to clear all condition
flags when bouncing a floating-point compare instruction. The
new support code only ORR the result flags on top of the
original flags without clearing any flags. This does not make
any sense to me, because clearing flags is also part of the
compare result. So we clear these bits here to make it PASS
*/
if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V))
fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V);
fpscr |= exceptions;
fmxr(FPSCR, fpscr);
#define RAISE(stat,en,sig) \
if (exceptions & stat && fpscr & en) \
si_code = sig;
/*
* These are arranged in priority order, least to highest.
*/
RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV);
RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES);
RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND);
RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
RAISE(FPSCR_IDC, FPSCR_IDE, FPE_FLTISN);
if (si_code)
vfp_raise_sigfpe(si_code, regs);
}
void vfp_pm_save_context(void)
{
u32 fpexc = fmrx(FPEXC);
unsigned int cpu = get_cpu();
/* Save last_VFP_context if needed */
if (last_VFP_context[cpu]) {
/* Enable vfp to save context */
if (!(fpexc & FPEXC_EN)) {
vfp_enable(NULL);
fmxr(FPEXC, fpexc | FPEXC_EN);
}
vfp_save_state(last_VFP_context[cpu], fpexc);
/* disable, just in case */
fmxr(FPEXC, fpexc & ~FPEXC_EN);
last_VFP_context[cpu] = NULL;
}
put_cpu();
}
static int vfp_pm_resume(struct sys_device *dev)
{
unsigned int cpu_arch = cpu_architecture();
/* ensure we have access to the vfp */
if (cpu_arch >= CPU_ARCH_ARMv6) {
vfp_enable(NULL);
}
/* and disable it to ensure the next usage restores the state */
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
return 0;
}
static int vfp_pm_suspend(void)
{
struct thread_info *ti = current_thread_info();
u32 fpexc = fmrx(FPEXC);
/* if vfp is on, then save state for resumption */
if (fpexc & FPEXC_EN) {
printk(KERN_DEBUG "%s: saving vfp state\n", __func__);
vfp_save_state(&ti->vfpstate, fpexc);
/* disable, just in case */
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
} else if (vfp_current_hw_state[ti->cpu]) {
fmxr(FPEXC, fpexc | FPEXC_EN);
vfp_save_state(vfp_current_hw_state[ti->cpu], fpexc);
fmxr(FPEXC, fpexc);
}
/* clear any information we had about last context state */
memset(vfp_current_hw_state, 0, sizeof(vfp_current_hw_state));
return 0;
}
static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd,
void *v)
{
u32 fpexc = fmrx(FPEXC);
unsigned int cpu = smp_processor_id();
switch (cmd) {
case CPU_PM_ENTER:
if (vfp_current_hw_state[cpu]) {
fmxr(FPEXC, fpexc | FPEXC_EN);
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
/* force a reload when coming back from idle */
vfp_current_hw_state[cpu] = NULL;
fmxr(FPEXC, fpexc & ~FPEXC_EN);
}
break;
case CPU_PM_ENTER_FAILED:
case CPU_PM_EXIT:
/* make sure VFP is disabled when leaving idle */
fmxr(FPEXC, fpexc & ~FPEXC_EN);
break;
}
return NOTIFY_OK;
}
int vfp_flush_context(void)
{
struct thread_info *ti = current_thread_info();
u32 fpexc = fmrx(FPEXC);
u32 cpu = ti->cpu;
int saved = 0;
#ifdef CONFIG_SMP
/* On SMP, if VFP is enabled, save the old state */
if ((fpexc & FPEXC_EN) && last_VFP_context[cpu]) {
#else
/* If there is a VFP context we must save it. */
if (last_VFP_context[cpu]) {
/* Enable VFP so we can save the old state. */
fmxr(FPEXC, fpexc | FPEXC_EN);
isb();
#endif
vfp_save_state(last_VFP_context[cpu], fpexc);
/* disable, just in case */
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
last_VFP_context[cpu] = NULL;
saved = 1;
}
return saved;
}
void vfp_reinit(void)
{
/* ensure we have access to the vfp */
vfp_enable(NULL);
/* and disable it to ensure the next usage restores the state */
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
}
/*
* When this function is called with the following 'cmd's, the following
* is true while this function is being run:
* THREAD_NOFTIFY_SWTICH:
* - the previously running thread will not be scheduled onto another CPU.
* - the next thread to be run (v) will not be running on another CPU.
* - thread->cpu is the local CPU number
* - not preemptible as we're called in the middle of a thread switch
* THREAD_NOTIFY_FLUSH:
* - the thread (v) will be running on the local CPU, so
* v === current_thread_info()
* - thread->cpu is the local CPU number at the time it is accessed,
* but may change at any time.
* - we could be preempted if tree preempt rcu is enabled, so
* it is unsafe to use thread->cpu.
* THREAD_NOTIFY_EXIT
* - the thread (v) will be running on the local CPU, so
* v === current_thread_info()
* - thread->cpu is the local CPU number at the time it is accessed,
* but may change at any time.
* - we could be preempted if tree preempt rcu is enabled, so
* it is unsafe to use thread->cpu.
*/
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
{
struct thread_info *thread = v;
u32 fpexc;
#ifdef CONFIG_SMP
unsigned int cpu;
#endif
switch (cmd) {
case THREAD_NOTIFY_SWITCH:
fpexc = fmrx(FPEXC);
#ifdef CONFIG_SMP
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu])
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
#endif
/*
* Always disable VFP so we can lazily save/restore the
* old state.
*/
fmxr(FPEXC, fpexc & ~FPEXC_EN);
break;
case THREAD_NOTIFY_FLUSH:
vfp_thread_flush(thread);
break;
case THREAD_NOTIFY_EXIT:
vfp_thread_exit(thread);
break;
// 2017-09-09
case THREAD_NOTIFY_COPY:
vfp_thread_copy(thread);
break;
}
return NOTIFY_DONE;
}
static void vfp_thread_flush(struct thread_info *thread)
{
union vfp_state *vfp = &thread->vfpstate;
unsigned int cpu;
cpu = get_cpu();
if (vfp_current_hw_state[cpu] == vfp)
vfp_current_hw_state[cpu] = NULL;
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
put_cpu();
memset(vfp, 0, sizeof(union vfp_state));
vfp->hard.fpexc = FPEXC_EN;
vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
#ifdef CONFIG_SMP
vfp->hard.cpu = NR_CPUS;
#endif
}
static int vfp_pm_suspend(struct sys_device *dev, pm_message_t state)
{
struct thread_info *ti = current_thread_info();
u32 fpexc = fmrx(FPEXC);
/* if vfp is on, then save state for resumption */
if (fpexc & FPEXC_EN) {
printk(KERN_DEBUG "%s: saving vfp state\n", __func__);
vfp_save_state(&ti->vfpstate, fpexc);
/* disable, just in case */
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
}
/* clear any information we had about last context state */
memset(last_VFP_context, 0, sizeof(last_VFP_context));
return 0;
}
/*
* When this function is called with the following 'cmd's, the following
* is true while this function is being run:
* THREAD_NOFTIFY_SWTICH:
* - the previously running thread will not be scheduled onto another CPU.
* - the next thread to be run (v) will not be running on another CPU.
* - thread->cpu is the local CPU number
* - not preemptible as we're called in the middle of a thread switch
* THREAD_NOTIFY_FLUSH:
* - the thread (v) will be running on the local CPU, so
* v === current_thread_info()
* - thread->cpu is the local CPU number at the time it is accessed,
* but may change at any time.
* - we could be preempted if tree preempt rcu is enabled, so
* it is unsafe to use thread->cpu.
* THREAD_NOTIFY_EXIT
* - the thread (v) will be running on the local CPU, so
* v === current_thread_info()
* - thread->cpu is the local CPU number at the time it is accessed,
* but may change at any time.
* - we could be preempted if tree preempt rcu is enabled, so
* it is unsafe to use thread->cpu.
*/
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
{
struct thread_info *thread = v;
if (likely(cmd == THREAD_NOTIFY_SWITCH)) {
u32 fpexc = fmrx(FPEXC);
#ifdef CONFIG_SMP
unsigned int cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && last_VFP_context[cpu]) {
vfp_save_state(last_VFP_context[cpu], fpexc);
last_VFP_context[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
last_VFP_context[cpu] = NULL;
#endif
/*
* Always disable VFP so we can lazily save/restore the
* old state.
*/
fmxr(FPEXC, fpexc & ~FPEXC_EN);
return NOTIFY_DONE;
}
if (cmd == THREAD_NOTIFY_FLUSH)
vfp_thread_flush(thread);
else
vfp_thread_exit(thread);
return NOTIFY_DONE;
}
/*
* Process bitmask of exception conditions.
*/
static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs)
{
int si_code = 0;
pr_debug("VFP: raising exceptions %08x\n", exceptions);
if (exceptions == VFP_EXCEPTION_ERROR) {
vfp_panic("unhandled bounce", inst);
vfp_raise_sigfpe(0, regs);
return;
}
/*
* If any of the status flags are set, update the FPSCR.
* Comparison instructions always return at least one of
* these flags set.
*/
if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V))
fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V);
fpscr |= exceptions;
fmxr(FPSCR, fpscr);
#define RAISE(stat,en,sig) \
if (exceptions & stat && fpscr & en) \
si_code = sig;
/*
* These are arranged in priority order, least to highest.
*/
RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV);
RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES);
RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND);
RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
RAISE(FPSCR_IDC, FPSCR_IDE, FPE_FLTISN);
if (si_code)
vfp_raise_sigfpe(si_code, regs);
}
/*
* Per-thread VFP initialization.
*/
static void vfp_thread_flush(struct thread_info *thread)
{
union vfp_state *vfp = &thread->vfpstate;
unsigned int cpu;
memset(vfp, 0, sizeof(union vfp_state));
vfp->hard.fpexc = FPEXC_EN;
vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
/*
* Disable VFP to ensure we initialize it first. We must ensure
* that the modification of vfp_current_hw_state[] and hardware disable
* are done for the same CPU and without preemption.
*/
cpu = get_cpu();
if (vfp_current_hw_state[cpu] == vfp)
vfp_current_hw_state[cpu] = NULL;
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
put_cpu();
}
void vfp_flush_hwstate(struct thread_info *thread)
{
unsigned int cpu = get_cpu();
/*
* If the thread we're interested in is the current owner of the
* hardware VFP state, then we need to save its state.
*/
if (last_VFP_context[cpu] == &thread->vfpstate) {
u32 fpexc = fmrx(FPEXC);
fmxr(FPEXC, fpexc & ~FPEXC_EN);
/*
* Set the context to NULL to force a reload the next time
* the thread uses the VFP.
*/
last_VFP_context[cpu] = NULL;
}
put_cpu();
}
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
{
struct thread_info *thread = v;
u32 fpexc;
#ifdef CONFIG_SMP
unsigned int cpu;
#endif
switch (cmd) {
case THREAD_NOTIFY_SWITCH:
fpexc = fmrx(FPEXC);
#ifdef CONFIG_SMP
cpu = thread->cpu;
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu])
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
#endif
fmxr(FPEXC, fpexc & ~FPEXC_EN);
break;
case THREAD_NOTIFY_FLUSH:
vfp_thread_flush(thread);
break;
case THREAD_NOTIFY_EXIT:
vfp_thread_exit(thread);
break;
case THREAD_NOTIFY_COPY:
vfp_thread_copy(thread);
break;
}
return NOTIFY_DONE;
}
void kernel_neon_end(void)
{
/* Disable the NEON/VFP unit. */
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
put_cpu();
}
/*
* Package up a bounce condition.
*/
void VFP9_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
{
u32 fpscr, orig_fpscr, exceptions, inst;
pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
/*
* Enable access to the VFP so we can handle the bounce.
*/
fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_INV|FPEXC_UFC|FPEXC_IOC));
orig_fpscr = fpscr = fmrx(FPSCR);
/*
* If we are running with inexact exceptions enabled, we need to
* emulate the trigger instruction. Note that as we're emulating
* the trigger instruction, we need to increment PC.
*/
if (fpscr & FPSCR_IXE) {
regs->ARM_pc += 4;
goto emulate;
}
barrier();
/*
* Modify fpscr to indicate the number of iterations remaining
*/
if (fpexc & FPEXC_EX) {
u32 len;
len = fpexc + (1 << FPEXC_LENGTH_BIT);
fpscr &= ~FPSCR_LENGTH_MASK;
fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT);
}
/*
* Handle the first FP instruction. We used to take note of the
* FPEXC bounce reason, but this appears to be unreliable.
* Emulate the bounced instruction instead.
*/
#ifndef CONFIG_VFPv3
inst = fmrx(FPINST);
#else
inst = trigger;
#endif
exceptions = vfp_emulate_instruction(inst, fpscr, regs);
if (exceptions)
vfp_raise_exceptions(exceptions, inst, orig_fpscr, regs);
#ifndef CONFIG_VFPv3
/*
* If there isn't a second FP instruction, exit now.
*/
if (!(fpexc & FPEXC_FPV2))
return;
/*
* The barrier() here prevents fpinst2 being read
* before the condition above.
*/
barrier();
trigger = fmrx(FPINST2);
orig_fpscr = fpscr = fmrx(FPSCR);
#else
return;
#endif
emulate:
exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
if (exceptions)
vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
}
/*
* Package up a bounce condition.
*/
void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
{
u32 fpscr, orig_fpscr, fpsid, exceptions;
pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
/*
* At this point, FPEXC can have the following configuration:
*
* EX DEX IXE
* 0 1 x - synchronous exception
* 1 x 0 - asynchronous exception
* 1 x 1 - sychronous on VFP subarch 1 and asynchronous on later
* 0 0 1 - synchronous on VFP9 (non-standard subarch 1
* implementation), undefined otherwise
*
* Clear various bits and enable access to the VFP so we can
* handle the bounce.
*/
fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK));
fpsid = fmrx(FPSID);
orig_fpscr = fpscr = fmrx(FPSCR);
/*
* Check for the special VFP subarch 1 and FPSCR.IXE bit case
*/
if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT)
&& (fpscr & FPSCR_IXE)) {
/*
* Synchronous exception, emulate the trigger instruction
*/
goto emulate;
}
if (fpexc & FPEXC_EX) {
#ifndef CONFIG_CPU_FEROCEON
/*
* Asynchronous exception. The instruction is read from FPINST
* and the interrupted instruction has to be restarted.
*/
trigger = fmrx(FPINST);
regs->ARM_pc -= 4;
#endif
} else if (!(fpexc & FPEXC_DEX)) {
/*
* Illegal combination of bits. It can be caused by an
* unallocated VFP instruction but with FPSCR.IXE set and not
* on VFP subarch 1.
*/
vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs);
goto exit;
}
/*
* Modify fpscr to indicate the number of iterations remaining.
* If FPEXC.EX is 0, FPEXC.DEX is 1 and the FPEXC.VV bit indicates
* whether FPEXC.VECITR or FPSCR.LEN is used.
*/
if (fpexc & (FPEXC_EX | FPEXC_VV)) {
u32 len;
len = fpexc + (1 << FPEXC_LENGTH_BIT);
fpscr &= ~FPSCR_LENGTH_MASK;
fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT);
}
/*
* Handle the first FP instruction. We used to take note of the
* FPEXC bounce reason, but this appears to be unreliable.
* Emulate the bounced instruction instead.
*/
exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
if (exceptions)
vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
/*
* If there isn't a second FP instruction, exit now. Note that
* the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
*/
if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
goto exit;
/*
* The barrier() here prevents fpinst2 being read
* before the condition above.
*/
barrier();
trigger = fmrx(FPINST2);
emulate:
exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
if (exceptions)
vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
exit:
preempt_enable();
}
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
{
struct thread_info *thread = v;
union vfp_state *vfp;
unsigned long flags;
__u32 cpu = thread->cpu;
if (likely(cmd == THREAD_NOTIFY_SWITCH)) {
u32 fpexc;
local_irq_save_hw_cond(flags);
fpexc = fmrx(FPEXC);
#ifdef CONFIG_SMP
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && last_VFP_context[cpu]) {
vfp_save_state(last_VFP_context[cpu], fpexc);
last_VFP_context[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
last_VFP_context[cpu] = NULL;
#endif
/*
* Always disable VFP so we can lazily save/restore the
* old state.
*/
fmxr(FPEXC, fpexc & ~FPEXC_EN);
local_irq_restore_hw_cond(flags);
return NOTIFY_DONE;
}
vfp = &thread->vfpstate;
if (cmd == THREAD_NOTIFY_FLUSH) {
/*
* Per-thread VFP initialisation.
*/
memset(vfp, 0, sizeof(union vfp_state));
vfp->hard.fpexc = FPEXC_EN;
vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
/*
* Disable VFP to ensure we initialise it first.
*/
local_irq_save_hw_cond(flags);
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
} else
local_irq_save_hw_cond(flags);
/* flush and release case: Per-thread VFP cleanup. */
if (last_VFP_context[cpu] == vfp)
last_VFP_context[cpu] = NULL;
local_irq_restore_hw_cond(flags);
return NOTIFY_DONE;
}
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
{
struct thread_info *thread = v;
union vfp_state *vfp;
__u32 cpu = thread->cpu;
if (likely(cmd == THREAD_NOTIFY_SWITCH)) {
u32 fpexc = fmrx(FPEXC);
#ifdef CONFIG_SMP
/*
* RCU locking is needed in case last_VFP_context[cpu] is
* released on a different CPU.
*/
rcu_read_lock();
vfp = last_VFP_context[cpu];
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp) {
vfp_save_state(vfp, fpexc);
vfp->hard.cpu = cpu;
}
rcu_read_unlock();
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
last_VFP_context[cpu] = NULL;
#endif
/*
* Always disable VFP so we can lazily save/restore the
* old state.
*/
fmxr(FPEXC, fpexc & ~FPEXC_EN);
return NOTIFY_DONE;
}
vfp = &thread->vfpstate;
if (cmd == THREAD_NOTIFY_FLUSH) {
/*
* Per-thread VFP initialisation.
*/
memset(vfp, 0, sizeof(union vfp_state));
vfp->hard.fpexc = FPEXC_EN;
vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
/*
* Disable VFP to ensure we initialise it first.
*/
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
}
/* flush and release case: Per-thread VFP cleanup. */
#ifndef CONFIG_SMP
if (last_VFP_context[cpu] == vfp)
last_VFP_context[cpu] = NULL;
#else
/*
* Since release_thread() may be called from a different CPU, we use
* cmpxchg() here to avoid a race with the vfp_support_entry() code
* which modifies last_VFP_context[cpu]. Note that on SMP systems, a
* STR instruction on a different CPU clears the global exclusive
* monitor state.
*/
(void)cmpxchg(&last_VFP_context[cpu], vfp, NULL);
#endif
return NOTIFY_DONE;
}