/*
* Read the IA32_APERF and IA32_MPERF counters. The first
* increments at the rate of the fixed maximum frequency
* configured during the boot, whereas APERF counts at the
* rate of the actual frequency. Note that the MSRs must be
* read without delay, and that only the ratio between
* IA32_APERF and IA32_MPERF is architecturally defined.
*
* The function thus returns the percentage of the actual
* frequency in terms of the maximum frequency of the calling
* CPU since the last call. A value zero implies an error.
*
* For further details, refer to:
*
* Intel Corporation: Intel 64 and IA-32 Architectures
* Software Developer's Manual. Section 13.2, Volume 3A:
* System Programming Guide, Part 1. July, 2008.
*
* Advanced Micro Devices: BIOS and Kernel Developer's
* Guide (BKDG) for AMD Family 10h Processors. Section
* 2.4.5, Revision 3.48, April 2010.
*/
uint8_t
acpicpu_md_pstate_hwf(struct cpu_info *ci)
{
struct acpicpu_softc *sc;
uint64_t aperf, mperf;
uint8_t rv = 0;
sc = acpicpu_sc[ci->ci_acpiid];
if (__predict_false(sc == NULL))
return 0;
if (__predict_false((sc->sc_flags & ACPICPU_FLAG_P_HWF) == 0))
return 0;
aperf = sc->sc_pstate_aperf;
mperf = sc->sc_pstate_mperf;
x86_disable_intr();
sc->sc_pstate_aperf = rdmsr(MSR_APERF);
sc->sc_pstate_mperf = rdmsr(MSR_MPERF);
x86_enable_intr();
aperf = sc->sc_pstate_aperf - aperf;
mperf = sc->sc_pstate_mperf - mperf;
if (__predict_true(mperf != 0))
rv = (aperf * 100) / mperf;
return rv;
}
/*
* Called with interrupts enabled.
*/
void
acpicpu_md_cstate_enter(int method, int state)
{
struct cpu_info *ci = curcpu();
KASSERT(ci->ci_ilevel == IPL_NONE);
switch (method) {
case ACPICPU_C_STATE_FFH:
x86_monitor(&ci->ci_want_resched, 0, 0);
if (__predict_false(ci->ci_want_resched != 0))
return;
x86_mwait((state - 1) << 4, 0);
break;
case ACPICPU_C_STATE_HALT:
x86_disable_intr();
if (__predict_false(ci->ci_want_resched != 0)) {
x86_enable_intr();
return;
}
x86_stihlt();
break;
}
}
void
fputrap(struct trapframe *frame)
{
register struct lwp *l = curcpu()->ci_fpcurlwp;
struct pcb *pcb = lwp_getpcb(l);
struct savefpu *sfp = &pcb->pcb_savefpu;
uint32_t mxcsr, statbits;
uint16_t cw;
ksiginfo_t ksi;
KPREEMPT_DISABLE(l);
x86_enable_intr();
/*
* At this point, fpcurlwp should be curlwp. If it wasn't, the TS bit
* should be set, and we should have gotten a DNA exception.
*/
KASSERT(l == curlwp);
fxsave(sfp);
pcb->pcb_savefpu_i387.fp_ex_tw = sfp->fp_fxsave.fx_ftw;
pcb->pcb_savefpu_i387.fp_ex_sw = sfp->fp_fxsave.fx_fsw;
if (frame->tf_trapno == T_XMM) {
mxcsr = sfp->fp_fxsave.fx_mxcsr;
statbits = mxcsr;
mxcsr &= ~0x3f;
x86_ldmxcsr(&mxcsr);
} else {
fninit();
fwait();
cw = sfp->fp_fxsave.fx_fcw;
fldcw(&cw);
fwait();
statbits = sfp->fp_fxsave.fx_fsw;
}
KPREEMPT_ENABLE(l);
KSI_INIT_TRAP(&ksi);
ksi.ksi_signo = SIGFPE;
ksi.ksi_addr = (void *)frame->tf_rip;
ksi.ksi_code = x86fpflags_to_ksiginfo(statbits);
ksi.ksi_trap = statbits;
(*l->l_proc->p_emul->e_trapsignal)(l, &ksi);
}
static void
acpicpu_md_pstate_hwf_reset(void *arg1, void *arg2)
{
struct cpu_info *ci = curcpu();
struct acpicpu_softc *sc;
sc = acpicpu_sc[ci->ci_acpiid];
if (__predict_false(sc == NULL))
return;
x86_disable_intr();
wrmsr(MSR_APERF, 0);
wrmsr(MSR_MPERF, 0);
x86_enable_intr();
sc->sc_pstate_aperf = 0;
sc->sc_pstate_mperf = 0;
}
/*
* trap(frame): exception, fault, and trap interface to BSD kernel.
*
* This common code is called from assembly language IDT gate entry routines
* that prepare a suitable stack frame, and restore this frame after the
* exception has been processed. Note that the effect is as if the arguments
* were passed call by reference.
*/
void
trap(struct trapframe *frame)
{
struct lwp *l = curlwp;
struct proc *p;
struct pcb *pcb;
extern char fusubail[], kcopy_fault[], return_address_fault[],
IDTVEC(osyscall)[];
struct trapframe *vframe;
ksiginfo_t ksi;
void *onfault;
int type, error;
uint32_t cr2;
bool pfail;
if (__predict_true(l != NULL)) {
pcb = lwp_getpcb(l);
p = l->l_proc;
} else {
/*
* this can happen eg. on break points in early on boot.
*/
pcb = NULL;
p = NULL;
}
type = frame->tf_trapno;
#ifdef DEBUG
if (trapdebug) {
trap_print(frame, l);
}
#endif
if (type != T_NMI &&
!KERNELMODE(frame->tf_cs, frame->tf_eflags)) {
type |= T_USER;
l->l_md.md_regs = frame;
pcb->pcb_cr2 = 0;
LWP_CACHE_CREDS(l, p);
}
#ifdef KDTRACE_HOOKS
/*
* A trap can occur while DTrace executes a probe. Before
* executing the probe, DTrace blocks re-scheduling and sets
* a flag in its per-cpu flags to indicate that it doesn't
* want to fault. On returning from the the probe, the no-fault
* flag is cleared and finally re-scheduling is enabled.
*
* If the DTrace kernel module has registered a trap handler,
* call it and if it returns non-zero, assume that it has
* handled the trap and modified the trap frame so that this
* function can return normally.
*/
if ((type == T_PROTFLT || type == T_PAGEFLT) &&
dtrace_trap_func != NULL) {
if ((*dtrace_trap_func)(frame, type)) {
return;
}
}
#endif
switch (type) {
case T_ASTFLT:
/*FALLTHROUGH*/
default:
we_re_toast:
if (type == T_TRCTRAP)
check_dr0();
else
trap_print(frame, l);
if (kdb_trap(type, 0, frame))
return;
if (kgdb_trap(type, frame))
return;
/*
* If this is a breakpoint, don't panic if we're not connected.
*/
if (type == T_BPTFLT && kgdb_disconnected()) {
printf("kgdb: ignored %s\n", trap_type[type]);
return;
}
panic("trap");
/*NOTREACHED*/
case T_PROTFLT:
case T_SEGNPFLT:
case T_ALIGNFLT:
case T_TSSFLT:
if (p == NULL)
goto we_re_toast;
/* Check for copyin/copyout fault. */
onfault = onfault_handler(pcb, frame);
if (onfault != NULL) {
copyefault:
error = EFAULT;
copyfault:
frame->tf_eip = (uintptr_t)onfault;
frame->tf_eax = error;
return;
}
/*
* Check for failure during return to user mode.
* This can happen loading invalid values into the segment
* registers, or during the 'iret' itself.
*
* We do this by looking at the instruction we faulted on.
* The specific instructions we recognize only happen when
* returning from a trap, syscall, or interrupt.
*/
kernelfault:
KSI_INIT_TRAP(&ksi);
ksi.ksi_signo = SIGSEGV;
ksi.ksi_code = SEGV_ACCERR;
ksi.ksi_trap = type;
switch (*(u_char *)frame->tf_eip) {
case 0xcf: /* iret */
/*
* The 'iret' instruction faulted, so we have the
* 'user' registers saved after the kernel %eip:%cs:%fl
* of the 'iret' and below that the user %eip:%cs:%fl
* the 'iret' was processing.
* We must delete the 3 words of kernel return address
* from the stack to generate a normal stack frame
* (eg for sending a SIGSEGV).
*/
vframe = (void *)((int *)frame + 3);
if (KERNELMODE(vframe->tf_cs, vframe->tf_eflags))
goto we_re_toast;
memmove(vframe, frame,
offsetof(struct trapframe, tf_eip));
/* Set the faulting address to the user %eip */
ksi.ksi_addr = (void *)vframe->tf_eip;
break;
case 0x8e:
switch (*(uint32_t *)frame->tf_eip) {
case 0x8e242c8e: /* mov (%esp,%gs), then */
case 0x0424648e: /* mov 0x4(%esp),%fs */
case 0x0824448e: /* mov 0x8(%esp),%es */
case 0x0c245c8e: /* mov 0xc(%esp),%ds */
break;
default:
goto we_re_toast;
}
/*
* We faulted loading one if the user segment registers.
* The stack frame containing the user registers is
* still valid and is just below the %eip:%cs:%fl of
* the kernel fault frame.
*/
vframe = (void *)(&frame->tf_eflags + 1);
if (KERNELMODE(vframe->tf_cs, vframe->tf_eflags))
goto we_re_toast;
/* There is no valid address for the fault */
break;
default:
goto we_re_toast;
}
/*
* We might have faulted trying to execute the
* trampoline for a local (nested) signal handler.
* Only generate SIGSEGV if the user %cs isn't changed.
* (This is only strictly necessary in the 'iret' case.)
*/
if (!pmap_exec_fixup(&p->p_vmspace->vm_map, vframe, pcb)) {
/* Save outer frame for any signal return */
l->l_md.md_regs = vframe;
(*p->p_emul->e_trapsignal)(l, &ksi);
}
/* Return to user by reloading the user frame */
trap_return_fault_return(vframe);
/* NOTREACHED */
case T_PROTFLT|T_USER: /* protection fault */
case T_TSSFLT|T_USER:
case T_SEGNPFLT|T_USER:
case T_STKFLT|T_USER:
case T_ALIGNFLT|T_USER:
KSI_INIT_TRAP(&ksi);
ksi.ksi_addr = (void *)rcr2();
switch (type) {
case T_SEGNPFLT|T_USER:
case T_STKFLT|T_USER:
ksi.ksi_signo = SIGBUS;
ksi.ksi_code = BUS_ADRERR;
break;
case T_TSSFLT|T_USER:
ksi.ksi_signo = SIGBUS;
ksi.ksi_code = BUS_OBJERR;
break;
case T_ALIGNFLT|T_USER:
ksi.ksi_signo = SIGBUS;
ksi.ksi_code = BUS_ADRALN;
break;
case T_PROTFLT|T_USER:
#ifdef VM86
if (frame->tf_eflags & PSL_VM) {
vm86_gpfault(l, type & ~T_USER);
goto out;
}
#endif
/*
* If pmap_exec_fixup does something,
* let's retry the trap.
*/
if (pmap_exec_fixup(&p->p_vmspace->vm_map, frame, pcb)){
goto out;
}
ksi.ksi_signo = SIGSEGV;
ksi.ksi_code = SEGV_ACCERR;
break;
default:
KASSERT(0);
break;
}
goto trapsignal;
case T_PRIVINFLT|T_USER: /* privileged instruction fault */
case T_FPOPFLT|T_USER: /* coprocessor operand fault */
KSI_INIT_TRAP(&ksi);
ksi.ksi_signo = SIGILL;
ksi.ksi_addr = (void *) frame->tf_eip;
switch (type) {
case T_PRIVINFLT|T_USER:
ksi.ksi_code = ILL_PRVOPC;
break;
case T_FPOPFLT|T_USER:
ksi.ksi_code = ILL_COPROC;
break;
default:
ksi.ksi_code = 0;
break;
}
goto trapsignal;
case T_ASTFLT|T_USER:
/* Allow process switch. */
//curcpu()->ci_data.cpu_nast++;
if (l->l_pflag & LP_OWEUPC) {
l->l_pflag &= ~LP_OWEUPC;
ADDUPROF(l);
}
/* Allow a forced task switch. */
if (curcpu()->ci_want_resched) {
preempt();
}
goto out;
case T_BOUND|T_USER:
case T_OFLOW|T_USER:
case T_DIVIDE|T_USER:
KSI_INIT_TRAP(&ksi);
ksi.ksi_signo = SIGFPE;
ksi.ksi_addr = (void *)frame->tf_eip;
switch (type) {
case T_BOUND|T_USER:
ksi.ksi_code = FPE_FLTSUB;
break;
case T_OFLOW|T_USER:
ksi.ksi_code = FPE_INTOVF;
break;
case T_DIVIDE|T_USER:
ksi.ksi_code = FPE_INTDIV;
break;
default:
ksi.ksi_code = 0;
break;
}
goto trapsignal;
case T_PAGEFLT:
/* Allow page faults in kernel mode. */
if (__predict_false(l == NULL))
goto we_re_toast;
/*
* fusubail is used by [fs]uswintr() to prevent page faulting
* from inside the profiling interrupt.
*/
onfault = pcb->pcb_onfault;
if (onfault == fusubail || onfault == return_address_fault) {
goto copyefault;
}
if (cpu_intr_p() || (l->l_pflag & LP_INTR) != 0) {
goto we_re_toast;
}
cr2 = rcr2();
goto faultcommon;
case T_PAGEFLT|T_USER: { /* page fault */
register vaddr_t va;
register struct vmspace *vm;
register struct vm_map *map;
vm_prot_t ftype;
extern struct vm_map *kernel_map;
cr2 = rcr2();
faultcommon:
vm = p->p_vmspace;
if (__predict_false(vm == NULL)) {
goto we_re_toast;
}
pcb->pcb_cr2 = cr2;
va = trunc_page((vaddr_t)cr2);
/*
* It is only a kernel address space fault iff:
* 1. (type & T_USER) == 0 and
* 2. pcb_onfault not set or
* 3. pcb_onfault set but supervisor space fault
* The last can occur during an exec() copyin where the
* argument space is lazy-allocated.
*/
if (type == T_PAGEFLT && va >= KERNBASE)
map = kernel_map;
else
map = &vm->vm_map;
if (frame->tf_err & PGEX_W)
ftype = VM_PROT_WRITE;
else if (frame->tf_err & PGEX_X)
ftype = VM_PROT_EXECUTE;
else
ftype = VM_PROT_READ;
#ifdef DIAGNOSTIC
if (map == kernel_map && va == 0) {
printf("trap: bad kernel access at %lx\n", va);
goto we_re_toast;
}
#endif
/* Fault the original page in. */
onfault = pcb->pcb_onfault;
pcb->pcb_onfault = NULL;
error = uvm_fault(map, va, ftype);
pcb->pcb_onfault = onfault;
if (error == 0) {
if (map != kernel_map && (void *)va >= vm->vm_maxsaddr)
uvm_grow(p, va);
pfail = false;
while (type == T_PAGEFLT) {
/*
* we need to switch pmap now if we're in
* the middle of copyin/out.
*
* but we don't need to do so for kcopy as
* it never touch userspace.
*/
kpreempt_disable();
if (curcpu()->ci_want_pmapload) {
onfault = onfault_handler(pcb, frame);
if (onfault != kcopy_fault) {
pmap_load();
}
}
/*
* We need to keep the pmap loaded and
* so avoid being preempted until back
* into the copy functions. Disable
* interrupts at the hardware level before
* re-enabling preemption. Interrupts
* will be re-enabled by 'iret' when
* returning back out of the trap stub.
* They'll only be re-enabled when the
* program counter is once again in
* the copy functions, and so visible
* to cpu_kpreempt_exit().
*/
#ifndef XEN
x86_disable_intr();
#endif
l->l_nopreempt--;
if (l->l_nopreempt > 0 || !l->l_dopreempt ||
pfail) {
return;
}
#ifndef XEN
x86_enable_intr();
#endif
/*
* If preemption fails for some reason,
* don't retry it. The conditions won't
* change under our nose.
*/
pfail = kpreempt(0);
}
goto out;
}
if (type == T_PAGEFLT) {
onfault = onfault_handler(pcb, frame);
if (onfault != NULL)
goto copyfault;
printf("uvm_fault(%p, %#lx, %d) -> %#x\n",
map, va, ftype, error);
goto kernelfault;
}
KSI_INIT_TRAP(&ksi);
ksi.ksi_trap = type & ~T_USER;
ksi.ksi_addr = (void *)cr2;
switch (error) {
case EINVAL:
ksi.ksi_signo = SIGBUS;
ksi.ksi_code = BUS_ADRERR;
break;
case EACCES:
ksi.ksi_signo = SIGSEGV;
ksi.ksi_code = SEGV_ACCERR;
error = EFAULT;
break;
case ENOMEM:
ksi.ksi_signo = SIGKILL;
printf("UVM: pid %d.%d (%s), uid %d killed: "
"out of swap\n", p->p_pid, l->l_lid, p->p_comm,
l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
break;
default:
ksi.ksi_signo = SIGSEGV;
ksi.ksi_code = SEGV_MAPERR;
break;
}
#ifdef TRAP_SIGDEBUG
printf("pid %d.%d (%s): signal %d at eip %x addr %lx "
"error %d\n", p->p_pid, l->l_lid, p->p_comm, ksi.ksi_signo,
frame->tf_eip, va, error);
#endif
(*p->p_emul->e_trapsignal)(l, &ksi);
break;
}
case T_TRCTRAP:
/* Check whether they single-stepped into a lcall. */
if (frame->tf_eip == (int)IDTVEC(osyscall))
return;
if (frame->tf_eip == (int)IDTVEC(osyscall) + 1) {
frame->tf_eflags &= ~PSL_T;
return;
}
goto we_re_toast;
case T_BPTFLT|T_USER: /* bpt instruction fault */
case T_TRCTRAP|T_USER: /* trace trap */
/*
* Don't go single-stepping into a RAS.
*/
if (p->p_raslist == NULL ||
(ras_lookup(p, (void *)frame->tf_eip) == (void *)-1)) {
KSI_INIT_TRAP(&ksi);
ksi.ksi_signo = SIGTRAP;
ksi.ksi_trap = type & ~T_USER;
if (type == (T_BPTFLT|T_USER))
ksi.ksi_code = TRAP_BRKPT;
else
ksi.ksi_code = TRAP_TRACE;
ksi.ksi_addr = (void *)frame->tf_eip;
(*p->p_emul->e_trapsignal)(l, &ksi);
}
break;
case T_NMI:
if (nmi_dispatch(frame))
return;
/* NMI can be hooked up to a pushbutton for debugging */
if (kgdb_trap(type, frame))
return;
if (kdb_trap(type, 0, frame))
return;
/* machine/parity/power fail/"kitchen sink" faults */
#if NMCA > 0
mca_nmi();
#endif
x86_nmi();
}
if ((type & T_USER) == 0)
return;
out:
userret(l);
return;
trapsignal:
ksi.ksi_trap = type & ~T_USER;
(*p->p_emul->e_trapsignal)(l, &ksi);
userret(l);
}
void
acpi_md_OsEnableInterrupt(void)
{
x86_enable_intr();
}
/*
* Implement device not available (DNA) exception
*
* If we were the last lwp to use the FPU, we can simply return.
* Otherwise, we save the previous state, if necessary, and restore
* our last saved state.
*/
void
fpudna(struct cpu_info *ci)
{
uint16_t cw;
uint32_t mxcsr;
struct lwp *l, *fl;
struct pcb *pcb;
int s;
if (ci->ci_fpsaving) {
/* Recursive trap. */
x86_enable_intr();
return;
}
/* Lock out IPIs and disable preemption. */
s = splhigh();
x86_enable_intr();
/* Save state on current CPU. */
l = ci->ci_curlwp;
pcb = lwp_getpcb(l);
fl = ci->ci_fpcurlwp;
if (fl != NULL) {
/*
* It seems we can get here on Xen even if we didn't
* switch lwp. In this case do nothing
*/
if (fl == l) {
KASSERT(pcb->pcb_fpcpu == ci);
clts();
splx(s);
return;
}
KASSERT(fl != l);
fpusave_cpu(true);
KASSERT(ci->ci_fpcurlwp == NULL);
}
/* Save our state if on a remote CPU. */
if (pcb->pcb_fpcpu != NULL) {
/* Explicitly disable preemption before dropping spl. */
KPREEMPT_DISABLE(l);
splx(s);
fpusave_lwp(l, true);
KASSERT(pcb->pcb_fpcpu == NULL);
s = splhigh();
KPREEMPT_ENABLE(l);
}
/*
* Restore state on this CPU, or initialize. Ensure that
* the entire update is atomic with respect to FPU-sync IPIs.
*/
clts();
ci->ci_fpcurlwp = l;
pcb->pcb_fpcpu = ci;
if ((l->l_md.md_flags & MDL_USEDFPU) == 0) {
fninit();
cw = pcb->pcb_savefpu.fp_fxsave.fx_fcw;
fldcw(&cw);
mxcsr = pcb->pcb_savefpu.fp_fxsave.fx_mxcsr;
x86_ldmxcsr(&mxcsr);
l->l_md.md_flags |= MDL_USEDFPU;
} else {
/*
* AMD FPU's do not restore FIP, FDP, and FOP on fxrstor,
* leaking other process's execution history. Clear them
* manually.
*/
static const double zero = 0.0;
int status;
/*
* Clear the ES bit in the x87 status word if it is currently
* set, in order to avoid causing a fault in the upcoming load.
*/
fnstsw(&status);
if (status & 0x80)
fnclex();
/*
* Load the dummy variable into the x87 stack. This mangles
* the x87 stack, but we don't care since we're about to call
* fxrstor() anyway.
*/
fldummy(&zero);
fxrstor(&pcb->pcb_savefpu);
}
KASSERT(ci == curcpu());
splx(s);
}
