/* * When this function is called with the following 'cmd's, the following * is true while this function is being run: * THREAD_NOFTIFY_SWTICH: * - the previously running thread will not be scheduled onto another CPU. * - the next thread to be run (v) will not be running on another CPU. * - thread->cpu is the local CPU number * - not preemptible as we're called in the middle of a thread switch * THREAD_NOTIFY_FLUSH: * - the thread (v) will be running on the local CPU, so * v === current_thread_info() * - thread->cpu is the local CPU number at the time it is accessed, * but may change at any time. * - we could be preempted if tree preempt rcu is enabled, so * it is unsafe to use thread->cpu. * THREAD_NOTIFY_EXIT * - the thread (v) will be running on the local CPU, so * v === current_thread_info() * - thread->cpu is the local CPU number at the time it is accessed, * but may change at any time. * - we could be preempted if tree preempt rcu is enabled, so * it is unsafe to use thread->cpu. */ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) { struct thread_info *thread = v; u32 fpexc; #ifdef CONFIG_SMP unsigned int cpu; #endif switch (cmd) { case THREAD_NOTIFY_SWITCH: fpexc = fmrx(FPEXC); #ifdef CONFIG_SMP cpu = thread->cpu; /* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The * restoring is done lazily. */ if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) vfp_save_state(vfp_current_hw_state[cpu], fpexc); #endif /* * Always disable VFP so we can lazily save/restore the * old state. */ fmxr(FPEXC, fpexc & ~FPEXC_EN); break; case THREAD_NOTIFY_FLUSH: vfp_thread_flush(thread); break; case THREAD_NOTIFY_EXIT: vfp_thread_exit(thread); break; case THREAD_NOTIFY_COPY: vfp_thread_copy(thread); break; } return NOTIFY_DONE; }
/* * Ensure that the VFP state stored in 'thread->vfpstate' is up to date * with the hardware state. */ void vfp_sync_hwstate(struct thread_info *thread) { unsigned int cpu = get_cpu(); if (vfp_state_in_hw(cpu, thread)) { u32 fpexc = fmrx(FPEXC); /* * Save the last VFP state on this CPU. */ fmxr(FPEXC, fpexc | FPEXC_EN); vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN); fmxr(FPEXC, fpexc); } put_cpu(); }
int vfp_flush_context(void) { unsigned long flags; struct thread_info *ti; u32 fpexc; u32 cpu; int saved = 0; local_irq_save(flags); ti = current_thread_info(); fpexc = fmrx(FPEXC); cpu = ti->cpu; #ifdef CONFIG_SMP /* On SMP, if VFP is enabled, save the old state */ if ((fpexc & FPEXC_EN) && last_VFP_context[cpu]) { last_VFP_context[cpu]->hard.cpu = cpu; #else /* If there is a VFP context we must save it. */ if (last_VFP_context[cpu]) { /* Enable VFP so we can save the old state. */ fmxr(FPEXC, fpexc | FPEXC_EN); isb(); #endif vfp_save_state(last_VFP_context[cpu], fpexc); /* disable, just in case */ fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); saved = 1; } last_VFP_context[cpu] = NULL; local_irq_restore(flags); return saved; } void vfp_reinit(void) { /* ensure we have access to the vfp */ vfp_enable(NULL); /* and disable it to ensure the next usage restores the state */ fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); }
/* * When this function is called with the following 'cmd's, the following * is true while this function is being run: * THREAD_NOFTIFY_SWTICH: * - the previously running thread will not be scheduled onto another CPU. * - the next thread to be run (v) will not be running on another CPU. * - thread->cpu is the local CPU number * - not preemptible as we're called in the middle of a thread switch * THREAD_NOTIFY_FLUSH: * - the thread (v) will be running on the local CPU, so * v === current_thread_info() * - thread->cpu is the local CPU number at the time it is accessed, * but may change at any time. * - we could be preempted if tree preempt rcu is enabled, so * it is unsafe to use thread->cpu. * THREAD_NOTIFY_EXIT * - the thread (v) will be running on the local CPU, so * v === current_thread_info() * - thread->cpu is the local CPU number at the time it is accessed, * but may change at any time. * - we could be preempted if tree preempt rcu is enabled, so * it is unsafe to use thread->cpu. */ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) { struct thread_info *thread = v; if (likely(cmd == THREAD_NOTIFY_SWITCH)) { u32 fpexc = fmrx(FPEXC); #ifdef CONFIG_SMP unsigned int cpu = thread->cpu; /* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The * restoring is done lazily. */ if ((fpexc & FPEXC_EN) && last_VFP_context[cpu]) { vfp_save_state(last_VFP_context[cpu], fpexc); last_VFP_context[cpu]->hard.cpu = cpu; } /* * Thread migration, just force the reloading of the * state on the new CPU in case the VFP registers * contain stale data. */ if (thread->vfpstate.hard.cpu != cpu) last_VFP_context[cpu] = NULL; #endif /* * Always disable VFP so we can lazily save/restore the * old state. */ fmxr(FPEXC, fpexc & ~FPEXC_EN); return NOTIFY_DONE; } if (cmd == THREAD_NOTIFY_FLUSH) vfp_thread_flush(thread); else vfp_thread_exit(thread); return NOTIFY_DONE; }
static int vfp_pm_suspend(struct sys_device *dev, pm_message_t state) { struct thread_info *ti = current_thread_info(); u32 fpexc = fmrx(FPEXC); /* if vfp is on, then save state for resumption */ if (fpexc & FPEXC_EN) { printk(KERN_DEBUG "%s: saving vfp state\n", __func__); vfp_save_state(&ti->vfpstate, fpexc); /* disable, just in case */ fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); } /* clear any information we had about last context state */ memset(last_VFP_context, 0, sizeof(last_VFP_context)); return 0; }
static int vfp_pm_suspend(void) { struct thread_info *ti = current_thread_info(); u32 fpexc = fmrx(FPEXC); /* if vfp is on, then save state for resumption */ if (fpexc & FPEXC_EN) { printk(KERN_DEBUG "%s: saving vfp state\n", __func__); vfp_save_state(&ti->vfpstate, fpexc); /* disable, just in case */ fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); } /* clear any information we had about last context state */ vfp_current_hw_state[ti->cpu] = NULL; return 0; }
void vfp_sync_hwstate(struct thread_info *thread) { unsigned int cpu = get_cpu(); /* * If the thread we're interested in is the current owner of the * hardware VFP state, then we need to save its state. */ if (vfp_current_hw_state[cpu] == &thread->vfpstate) { u32 fpexc = fmrx(FPEXC); /* * Save the last VFP state on this CPU. */ fmxr(FPEXC, fpexc | FPEXC_EN); vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN); fmxr(FPEXC, fpexc); } put_cpu(); }
int fill_fpregs(struct thread *td, struct fpreg *regs) { #ifdef VFP struct pcb *pcb; pcb = td->td_pcb; if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running VFP instructions we will * need to save the state to memcpy it below. */ vfp_save_state(td, pcb); memcpy(regs->fp_q, pcb->pcb_vfp, sizeof(regs->fp_q)); regs->fp_cr = pcb->pcb_fpcr; regs->fp_sr = pcb->pcb_fpsr; } else #endif memset(regs->fp_q, 0, sizeof(regs->fp_q)); return (0); }
void vfp_pm_save_context(void) { u32 fpexc = fmrx(FPEXC); unsigned int cpu = get_cpu(); /* Save last_VFP_context if needed */ if (last_VFP_context[cpu]) { /* Enable vfp to save context */ if (!(fpexc & FPEXC_EN)) { vfp_enable(NULL); fmxr(FPEXC, fpexc | FPEXC_EN); } vfp_save_state(last_VFP_context[cpu], fpexc); /* disable, just in case */ fmxr(FPEXC, fpexc & ~FPEXC_EN); last_VFP_context[cpu] = NULL; } put_cpu(); }
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) { struct thread_info *thread = v; u32 fpexc; #ifdef CONFIG_SMP unsigned int cpu; #endif switch (cmd) { case THREAD_NOTIFY_SWITCH: fpexc = fmrx(FPEXC); #ifdef CONFIG_SMP cpu = thread->cpu; if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) vfp_save_state(vfp_current_hw_state[cpu], fpexc); #endif fmxr(FPEXC, fpexc & ~FPEXC_EN); break; case THREAD_NOTIFY_FLUSH: vfp_thread_flush(thread); break; case THREAD_NOTIFY_EXIT: vfp_thread_exit(thread); break; case THREAD_NOTIFY_COPY: vfp_thread_copy(thread); break; } return NOTIFY_DONE; }
static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) { u32 fpexc = fmrx(FPEXC); unsigned int cpu = smp_processor_id(); switch (cmd) { case CPU_PM_ENTER: if (vfp_current_hw_state[cpu]) { fmxr(FPEXC, fpexc | FPEXC_EN); vfp_save_state(vfp_current_hw_state[cpu], fpexc); /* force a reload when coming back from idle */ vfp_current_hw_state[cpu] = NULL; fmxr(FPEXC, fpexc & ~FPEXC_EN); } break; case CPU_PM_ENTER_FAILED: case CPU_PM_EXIT: /* make sure VFP is disabled when leaving idle */ fmxr(FPEXC, fpexc & ~FPEXC_EN); break; } return NOTIFY_OK; }
static void ctxt_switch_from(struct vcpu *p) { p2m_save_state(p); /* CP 15 */ p->arch.csselr = READ_SYSREG(CSSELR_EL1); /* Control Registers */ p->arch.cpacr = READ_SYSREG(CPACR_EL1); p->arch.contextidr = READ_SYSREG(CONTEXTIDR_EL1); p->arch.tpidr_el0 = READ_SYSREG(TPIDR_EL0); p->arch.tpidrro_el0 = READ_SYSREG(TPIDRRO_EL0); p->arch.tpidr_el1 = READ_SYSREG(TPIDR_EL1); /* Arch timer */ p->arch.cntkctl = READ_SYSREG32(CNTKCTL_EL1); virt_timer_save(p); if ( is_32bit_domain(p->domain) && cpu_has_thumbee ) { p->arch.teecr = READ_SYSREG32(TEECR32_EL1); p->arch.teehbr = READ_SYSREG32(TEEHBR32_EL1); } #ifdef CONFIG_ARM_32 p->arch.joscr = READ_CP32(JOSCR); p->arch.jmcr = READ_CP32(JMCR); #endif isb(); /* MMU */ p->arch.vbar = READ_SYSREG(VBAR_EL1); p->arch.ttbcr = READ_SYSREG(TCR_EL1); p->arch.ttbr0 = READ_SYSREG64(TTBR0_EL1); p->arch.ttbr1 = READ_SYSREG64(TTBR1_EL1); if ( is_32bit_domain(p->domain) ) p->arch.dacr = READ_SYSREG(DACR32_EL2); p->arch.par = READ_SYSREG64(PAR_EL1); #if defined(CONFIG_ARM_32) p->arch.mair0 = READ_CP32(MAIR0); p->arch.mair1 = READ_CP32(MAIR1); p->arch.amair0 = READ_CP32(AMAIR0); p->arch.amair1 = READ_CP32(AMAIR1); #else p->arch.mair = READ_SYSREG64(MAIR_EL1); p->arch.amair = READ_SYSREG64(AMAIR_EL1); #endif /* Fault Status */ #if defined(CONFIG_ARM_32) p->arch.dfar = READ_CP32(DFAR); p->arch.ifar = READ_CP32(IFAR); p->arch.dfsr = READ_CP32(DFSR); #elif defined(CONFIG_ARM_64) p->arch.far = READ_SYSREG64(FAR_EL1); p->arch.esr = READ_SYSREG64(ESR_EL1); #endif if ( is_32bit_domain(p->domain) ) p->arch.ifsr = READ_SYSREG(IFSR32_EL2); p->arch.afsr0 = READ_SYSREG(AFSR0_EL1); p->arch.afsr1 = READ_SYSREG(AFSR1_EL1); /* XXX MPU */ /* VFP */ vfp_save_state(p); /* VGIC */ gic_save_state(p); isb(); context_saved(p); }
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) { struct thread_info *thread = v; union vfp_state *vfp; __u32 cpu = thread->cpu; if (likely(cmd == THREAD_NOTIFY_SWITCH)) { u32 fpexc = fmrx(FPEXC); #ifdef CONFIG_SMP /* * RCU locking is needed in case last_VFP_context[cpu] is * released on a different CPU. */ rcu_read_lock(); vfp = last_VFP_context[cpu]; /* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The * restoring is done lazily. */ if ((fpexc & FPEXC_EN) && vfp) { vfp_save_state(vfp, fpexc); vfp->hard.cpu = cpu; } rcu_read_unlock(); /* * Thread migration, just force the reloading of the * state on the new CPU in case the VFP registers * contain stale data. */ if (thread->vfpstate.hard.cpu != cpu) last_VFP_context[cpu] = NULL; #endif /* * Always disable VFP so we can lazily save/restore the * old state. */ fmxr(FPEXC, fpexc & ~FPEXC_EN); return NOTIFY_DONE; } vfp = &thread->vfpstate; if (cmd == THREAD_NOTIFY_FLUSH) { /* * Per-thread VFP initialisation. */ memset(vfp, 0, sizeof(union vfp_state)); vfp->hard.fpexc = FPEXC_EN; vfp->hard.fpscr = FPSCR_ROUND_NEAREST; /* * Disable VFP to ensure we initialise it first. */ fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); } /* flush and release case: Per-thread VFP cleanup. */ #ifndef CONFIG_SMP if (last_VFP_context[cpu] == vfp) last_VFP_context[cpu] = NULL; #else /* * Since release_thread() may be called from a different CPU, we use * cmpxchg() here to avoid a race with the vfp_support_entry() code * which modifies last_VFP_context[cpu]. Note that on SMP systems, a * STR instruction on a different CPU clears the global exclusive * monitor state. */ (void)cmpxchg(&last_VFP_context[cpu], vfp, NULL); #endif return NOTIFY_DONE; }
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) { struct thread_info *thread = v; union vfp_state *vfp; unsigned long flags; __u32 cpu = thread->cpu; if (likely(cmd == THREAD_NOTIFY_SWITCH)) { u32 fpexc; local_irq_save_hw_cond(flags); fpexc = fmrx(FPEXC); #ifdef CONFIG_SMP /* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The * restoring is done lazily. */ if ((fpexc & FPEXC_EN) && last_VFP_context[cpu]) { vfp_save_state(last_VFP_context[cpu], fpexc); last_VFP_context[cpu]->hard.cpu = cpu; } /* * Thread migration, just force the reloading of the * state on the new CPU in case the VFP registers * contain stale data. */ if (thread->vfpstate.hard.cpu != cpu) last_VFP_context[cpu] = NULL; #endif /* * Always disable VFP so we can lazily save/restore the * old state. */ fmxr(FPEXC, fpexc & ~FPEXC_EN); local_irq_restore_hw_cond(flags); return NOTIFY_DONE; } vfp = &thread->vfpstate; if (cmd == THREAD_NOTIFY_FLUSH) { /* * Per-thread VFP initialisation. */ memset(vfp, 0, sizeof(union vfp_state)); vfp->hard.fpexc = FPEXC_EN; vfp->hard.fpscr = FPSCR_ROUND_NEAREST; /* * Disable VFP to ensure we initialise it first. */ local_irq_save_hw_cond(flags); fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); } else local_irq_save_hw_cond(flags); /* flush and release case: Per-thread VFP cleanup. */ if (last_VFP_context[cpu] == vfp) last_VFP_context[cpu] = NULL; local_irq_restore_hw_cond(flags); return NOTIFY_DONE; }