/* * Enable and initialize the xsave feature. */ void __ref xsave_cntxt_init(void) { unsigned int eax, ebx, ecx, edx; cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx); pcntxt_mask = eax + ((u64)edx << 32); if ((pcntxt_mask & XSTATE_FPSSE) != XSTATE_FPSSE) { printk(KERN_ERR "FP/SSE not shown under xsave features 0x%llx\n", pcntxt_mask); BUG(); } /* * Support only the state known to OS. */ pcntxt_mask = pcntxt_mask & XCNTXT_MASK; xsave_init(); /* * Recompute the context size for enabled features */ cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx); xstate_size = ebx; prepare_fx_sw_frame(); setup_xstate_init(); printk(KERN_INFO "xsave/xrstor: enabled xstate_bv 0x%llx, " "cntxt size 0x%x\n", pcntxt_mask, xstate_size); }
/* * Calculate total size of enabled xstates in XCR0/xfeatures_mask. * * Note the SDM's wording here. "sub-function 0" only enumerates * the size of the *user* states. If we use it to size a buffer * that we use 'XSAVES' on, we could potentially overflow the * buffer because 'XSAVES' saves system states too. * * Note that we do not currently set any bits on IA32_XSS so * 'XCR0 | IA32_XSS == XCR0' for now. */ static unsigned int __init calculate_xstate_size(void) { unsigned int eax, ebx, ecx, edx; unsigned int calculated_xstate_size; if (!cpu_has_xsaves) { /* * - CPUID function 0DH, sub-function 0: * EBX enumerates the size (in bytes) required by * the XSAVE instruction for an XSAVE area * containing all the *user* state components * corresponding to bits currently set in XCR0. */ cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); calculated_xstate_size = ebx; } else { /* * - CPUID function 0DH, sub-function 1: * EBX enumerates the size (in bytes) required by * the XSAVES instruction for an XSAVE area * containing all the state components * corresponding to bits currently set in * XCR0 | IA32_XSS. */ cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); calculated_xstate_size = ebx; } return calculated_xstate_size; }
/* * Actually perform cpuid operation. */ static int cpuctl_do_cpuid_count(int cpu, cpuctl_cpuid_count_args_t *data, struct thread *td) { int is_bound = 0; int oldcpu; KASSERT(cpu >= 0 && cpu <= mp_maxid, ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); /* Explicitly clear cpuid data to avoid returning stale info. */ bzero(data->data, sizeof(data->data)); DPRINTF("[cpuctl,%d]: retrieving cpuid lev %#0x type %#0x for %d cpu\n", __LINE__, data->level, data->level_type, cpu); #ifdef __i386__ if (cpu_id == 0) return (ENODEV); #endif oldcpu = td->td_oncpu; is_bound = cpu_sched_is_bound(td); set_cpu(cpu, td); cpuid_count(data->level, data->level_type, data->data); restore_cpu(oldcpu, is_bound, td); return (0); }
static void cpuid_smp_cpuid(void *cmd_block) { struct cpuid_regs *cmd = (struct cpuid_regs *)cmd_block; cpuid_count(cmd->eax, cmd->ecx, &cmd->eax, &cmd->ebx, &cmd->ecx, &cmd->edx); }
/* * Record the offsets and sizes of various xstates contained * in the XSAVE state memory layout. */ static void __init setup_xstate_features(void) { u32 eax, ebx, ecx, edx, i; /* start at the beginnning of the "extended state" */ unsigned int last_good_offset = offsetof(struct xregs_state, extended_state_area); for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { if (!xfeature_enabled(i)) continue; cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); xstate_offsets[i] = ebx; xstate_sizes[i] = eax; /* * In our xstate size checks, we assume that the * highest-numbered xstate feature has the * highest offset in the buffer. Ensure it does. */ WARN_ONCE(last_good_offset > xstate_offsets[i], "x86/fpu: misordered xstate at %d\n", last_good_offset); last_good_offset = xstate_offsets[i]; printk(KERN_INFO "x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", i, ebx, i, eax); } }
int main(int argc, char *argv[]) { uint32_t leaf, counter; uint32_t eax, ebx, ecx, edx; openconsole(&dev_null_r, &dev_stdcon_w); if (argc < 2 || argc > 4) { printf("Usage: %s leaf [counter]\n", argv[0]); exit(1); } leaf = strtoul(argv[1], NULL, 0); counter = (argv > 2) ? strtoul(argv[2], NULL, 0) : 0; if (!cpu_has_eflag(EFLAGS_ID)) { printf("The CPUID instruction is not supported\n"); exit(1); } cpuid_count(leaf, counter, &eax, &ebx, &ecx, &edx); dump_reg("eax", eax); dump_reg("ebx", ebx); dump_reg("ecx", ecx); dump_reg("edx", edx); return 0; }
int detect_extended_topology_early(struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP unsigned int eax, ebx, ecx, edx; if (c->cpuid_level < 0xb) return -1; cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx); /* * check if the cpuid leaf 0xb is actually implemented. */ if (ebx == 0 || (LEAFB_SUBTYPE(ecx) != SMT_TYPE)) return -1; set_cpu_cap(c, X86_FEATURE_XTOPOLOGY); /* * initial apic id, which also represents 32-bit extended x2apic id. */ c->initial_apicid = edx; smp_num_siblings = LEVEL_MAX_SIBLINGS(ebx); #endif return 0; }
/* * Check for extended topology enumeration cpuid leaf 0xb and if it * exists, use it for populating initial_apicid and cpu topology * detection. */ int detect_extended_topology(struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP unsigned int eax, ebx, ecx, edx, sub_index; unsigned int ht_mask_width, core_plus_mask_width; unsigned int core_select_mask, core_level_siblings; if (detect_extended_topology_early(c) < 0) return -1; /* * Populate HT related information from sub-leaf level 0. */ cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx); core_level_siblings = smp_num_siblings = LEVEL_MAX_SIBLINGS(ebx); core_plus_mask_width = ht_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); sub_index = 1; do { cpuid_count(0xb, sub_index, &eax, &ebx, &ecx, &edx); /* * Check for the Core type in the implemented sub leaves. */ if (LEAFB_SUBTYPE(ecx) == CORE_TYPE) { core_level_siblings = LEVEL_MAX_SIBLINGS(ebx); core_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); break; } sub_index++; } while (LEAFB_SUBTYPE(ecx) != INVALID_TYPE); core_select_mask = (~(-1 << core_plus_mask_width)) >> ht_mask_width; c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, ht_mask_width) & core_select_mask; c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, core_plus_mask_width); /* * Reinit the apicid, now that we have extended initial_apicid. */ c->apicid = apic->phys_pkg_id(c->initial_apicid, 0); c->x86_max_cores = (core_level_siblings / smp_num_siblings); #endif return 0; }
static int xfeature_size(int xfeature_nr) { u32 eax, ebx, ecx, edx; CHECK_XFEATURE(xfeature_nr); cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); return eax; }
static int xfeature_uncompacted_offset(int xfeature_nr) { u32 eax, ebx, ecx, edx; CHECK_XFEATURE(xfeature_nr); cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); return ebx; }
static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, u32 index) { entry->function = function; entry->index = index; cpuid_count(entry->function, entry->index, &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); entry->flags = 0; }
void kvm_xstate_size_init(void) { unsigned int eax, ebx, ecx, edx; /* kvm only uses xstate_size if xsave is supported */ if (cpu_has_xsave) { cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx); kvm_xstate_size = ebx; } }
/* * Calculate total size of enabled xstates in XCR0/xfeatures_mask. */ static void __init init_xstate_size(void) { unsigned int eax, ebx, ecx, edx; int i; if (!cpu_has_xsaves) { cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); xstate_size = ebx; return; } xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE; for (i = 2; i < 64; i++) { if (test_bit(i, (unsigned long *)&xfeatures_mask)) { cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); xstate_size += eax; } } }
void vmmr0_xstate_size_init(void) { unsigned int eax, ebx, ecx, edx; /* vmmr0 only uses xstate_size if xsave is supported */ if (cpu_has_xsave) { cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx); vmmr0_xstate_size = ebx; BUG_ON(vmmr0_xstate_size > sizeof(union vmmr0_thread_xstate)); } }
/* * Enable and initialize the xsave feature. */ static void __init xstate_enable_boot_cpu(void) { unsigned int eax, ebx, ecx, edx; if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { WARN(1, KERN_ERR "XSTATE_CPUID missing\n"); return; } cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); pcntxt_mask = eax + ((u64)edx << 32); if ((pcntxt_mask & XSTATE_FPSSE) != XSTATE_FPSSE) { printk(KERN_ERR "FP/SSE not shown under xsave features 0x%llx\n", pcntxt_mask); BUG(); } /* * Support only the state known to OS. */ pcntxt_mask = pcntxt_mask & XCNTXT_MASK; xstate_enable(); /* * Recompute the context size for enabled features */ cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); xstate_size = ebx; update_regset_xstate_info(xstate_size, pcntxt_mask); prepare_fx_sw_frame(); setup_xstate_init(); printk(KERN_INFO "xsave/xrstor: enabled xstate_bv 0x%llx, " "cntxt size 0x%x\n", pcntxt_mask, xstate_size); }
/* * We could cache this like xstate_size[], but we only use * it here, so it would be a waste of space. */ static int xfeature_is_aligned(int xfeature_nr) { u32 eax, ebx, ecx, edx; CHECK_XFEATURE(xfeature_nr); cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); /* * The value returned by ECX[1] indicates the alignment * of state component i when the compacted format * of the extended region of an XSAVE area is used */ return !!(ecx & 2); }
/* * find out the number of processor cores on the die */ static int num_cpu_cores(struct cpuinfo_x86 *c) { unsigned int eax, ebx, ecx, edx; if (c->cpuid_level < 4) return 1; /* Intel has a non-standard dependency on %ecx for this CPUID level. */ cpuid_count(4, 0, &eax, &ebx, &ecx, &edx); if (eax & 0x1f) return ((eax >> 26) + 1); else return 1;
static int intel_num_cpu_cores(struct cpudata *cpu) { unsigned int eax, ebx, ecx, edx; if (cpu->cpuid_level < 4) return 1; /* intel.has a non-standard dependency on %ecx for this CPUID level. */ cpuid_count(cpu->number, 4, 0, &eax, &ebx, &ecx, &edx); if (eax & 0x1f) return (eax >> 26) + 1; else return 1;
/* * Record the offsets and sizes of various xstates contained * in the XSAVE state memory layout. * * ( Note that certain features might be non-present, for them * we'll have 0 offset and 0 size. ) */ static void __init setup_xstate_features(void) { u32 eax, ebx, ecx, edx, leaf; xfeatures_nr = fls64(xfeatures_mask); for (leaf = 2; leaf < xfeatures_nr; leaf++) { cpuid_count(XSTATE_CPUID, leaf, &eax, &ebx, &ecx, &edx); xstate_offsets[leaf] = ebx; xstate_sizes[leaf] = eax; printk(KERN_INFO "x86/fpu: xstate_offset[%d]: %04x, xstate_sizes[%d]: %04x\n", leaf, ebx, leaf, eax); } }
/* * Enable and initialize the xsave feature. * Called once per system bootup. */ void __init fpu__init_system_xstate(void) { unsigned int eax, ebx, ecx, edx; static int on_boot_cpu __initdata = 1; int err; WARN_ON_FPU(!on_boot_cpu); on_boot_cpu = 0; if (!cpu_has_xsave) { pr_info("x86/fpu: Legacy x87 FPU detected.\n"); return; } if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { WARN_ON_FPU(1); return; } cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); xfeatures_mask = eax + ((u64)edx << 32); if ((xfeatures_mask & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask); BUG(); } xfeatures_mask &= fpu__get_supported_xfeatures_mask(); /* Enable xstate instructions to be able to continue with initialization: */ fpu__init_cpu_xstate(); err = init_xstate_size(); if (err) { /* something went wrong, boot without any XSAVE support */ fpu__init_disable_system_xstate(); return; } update_regset_xstate_info(xstate_size, xfeatures_mask); fpu__init_prepare_fx_sw_frame(); setup_init_fpu_buf(); setup_xstate_comp(); pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", xfeatures_mask, xstate_size, cpu_has_xsaves ? "compacted" : "standard"); }
int __getcontextx_size(void) { u_int p[4]; if (xstate_sz == -1) { do_cpuid(1, p); if ((p[2] & CPUID2_OSXSAVE) != 0) { cpuid_count(0xd, 0x0, p); xstate_sz = p[1] - sizeof(struct savefpu); } else xstate_sz = 0; } return (sizeof(ucontext_t) + xstate_sz); }
void init_scattered_cpuid_features(struct cpuinfo_x86 *c) { u32 max_level; u32 regs[4]; const struct cpuid_bit *cb; static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_DTS, CR_EAX, 0, 0x00000006, 0 }, { X86_FEATURE_DTHERM, CR_EAX, 0, 0x00000006, 0 }, { X86_FEATURE_IDA, CR_EAX, 1, 0x00000006, 0 }, { X86_FEATURE_ARAT, CR_EAX, 2, 0x00000006, 0 }, { X86_FEATURE_PLN, CR_EAX, 4, 0x00000006, 0 }, { X86_FEATURE_PTS, CR_EAX, 6, 0x00000006, 0 }, { X86_FEATURE_APERFMPERF, CR_ECX, 0, 0x00000006, 0 }, { X86_FEATURE_EPB, CR_ECX, 3, 0x00000006, 0 }, { X86_FEATURE_XSAVEOPT, CR_EAX, 0, 0x0000000d, 1 }, { X86_FEATURE_CPB, CR_EDX, 9, 0x80000007, 0 }, { X86_FEATURE_HW_PSTATE, CR_EDX, 7, 0x80000007, 0 }, { X86_FEATURE_NPT, CR_EDX, 0, 0x8000000a, 0 }, { X86_FEATURE_LBRV, CR_EDX, 1, 0x8000000a, 0 }, { X86_FEATURE_SVML, CR_EDX, 2, 0x8000000a, 0 }, { X86_FEATURE_NRIPS, CR_EDX, 3, 0x8000000a, 0 }, { X86_FEATURE_TSCRATEMSR, CR_EDX, 4, 0x8000000a, 0 }, { X86_FEATURE_VMCBCLEAN, CR_EDX, 5, 0x8000000a, 0 }, { X86_FEATURE_FLUSHBYASID, CR_EDX, 6, 0x8000000a, 0 }, { X86_FEATURE_DECODEASSISTS, CR_EDX, 7, 0x8000000a, 0 }, { X86_FEATURE_PAUSEFILTER, CR_EDX,10, 0x8000000a, 0 }, { X86_FEATURE_PFTHRESHOLD, CR_EDX,12, 0x8000000a, 0 }, { 0, 0, 0, 0, 0 } }; for (cb = cpuid_bits; cb->feature; cb++) { /* Verify that the level is valid */ max_level = cpuid_eax(cb->level & 0xffff0000); if (max_level < cb->level || max_level > (cb->level | 0xffff)) continue; cpuid_count(cb->level, cb->sub_leaf, ®s[CR_EAX], ®s[CR_EBX], ®s[CR_ECX], ®s[CR_EDX]); if (regs[cb->reg] & (1 << cb->bit)) set_cpu_cap(c, cb->feature); } }
/* * Calculate the fpu save area size. */ static void fpuinit_bsp2(void) { u_int cp[4]; if (use_xsave) { cpuid_count(0xd, 0x0, cp); cpu_max_ext_state_size = cp[1]; /* * Reload the cpu_feature2, since we enabled OSXSAVE. */ do_cpuid(1, cp); cpu_feature2 = cp[2]; } else cpu_max_ext_state_size = sizeof(struct savefpu); }
/* * Enable and initialize the xsave feature. * Called once per system bootup. */ void __init fpu__init_system_xstate(void) { unsigned int eax, ebx, ecx, edx; static int on_boot_cpu = 1; WARN_ON_FPU(!on_boot_cpu); on_boot_cpu = 0; if (!cpu_has_xsave) { pr_info("x86/fpu: Legacy x87 FPU detected.\n"); return; } if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { WARN_ON_FPU(1); return; } cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); xfeatures_mask = eax + ((u64)edx << 32); if ((xfeatures_mask & XSTATE_FPSSE) != XSTATE_FPSSE) { pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask); BUG(); } /* Support only the state known to the OS: */ xfeatures_mask = xfeatures_mask & XCNTXT_MASK; /* Enable xstate instructions to be able to continue with initialization: */ fpu__init_cpu_xstate(); /* Recompute the context size for enabled features: */ init_xstate_size(); update_regset_xstate_info(xstate_size, xfeatures_mask); fpu__init_prepare_fx_sw_frame(); setup_init_fpu_buf(); setup_xstate_comp(); pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is 0x%x bytes, using '%s' format.\n", xfeatures_mask, xstate_size, cpu_has_xsaves ? "compacted" : "standard"); }
static u32 xstate_required_size(u64 xstate_bv) { int feature_bit = 0; u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; xstate_bv &= XSTATE_EXTEND_MASK; while (xstate_bv) { if (xstate_bv & 0x1) { u32 eax, ebx, ecx, edx; cpuid_count(0xD, feature_bit, &eax, &ebx, &ecx, &edx); ret = max(ret, eax + ebx); } xstate_bv >>= 1; feature_bit++; } return ret; }
static void __xstate_dump_leaves(void) { int i; u32 eax, ebx, ecx, edx; static int should_dump = 1; if (!should_dump) return; should_dump = 0; /* * Dump out a few leaves past the ones that we support * just in case there are some goodies up there */ for (i = 0; i < XFEATURE_MAX + 10; i++) { cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n", XSTATE_CPUID, i, eax, ebx, ecx, edx); } }
static u32 xstate_required_size(u64 xstate_bv, bool compacted) { int feature_bit = 0; u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; xstate_bv &= XFEATURE_MASK_EXTEND; while (xstate_bv) { if (xstate_bv & 0x1) { u32 eax, ebx, ecx, edx, offset; cpuid_count(0xD, feature_bit, &eax, &ebx, &ecx, &edx); offset = compacted ? ret : ebx; ret = max(ret, offset + eax); } xstate_bv >>= 1; feature_bit++; } return ret; }
/* * Record the offsets and sizes of different state managed by the xsave * memory layout. */ static void __init setup_xstate_features(void) { int eax, ebx, ecx, edx, leaf = 0x2; xstate_features = fls64(pcntxt_mask); xstate_offsets = alloc_bootmem(xstate_features * sizeof(int)); xstate_sizes = alloc_bootmem(xstate_features * sizeof(int)); do { cpuid_count(XSTATE_CPUID, leaf, &eax, &ebx, &ecx, &edx); if (eax == 0) break; xstate_offsets[leaf] = ebx; xstate_sizes[leaf] = eax; leaf++; } while (1); }
/* * Actually perform cpuid operation. */ static int cpuctl_do_cpuid(int cpu, cpuctl_cpuid_args_t *data, struct thread *td) { int is_bound = 0; int oldcpu; KASSERT(cpu >= 0 && cpu < mp_ncpus, ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); /* Explicitly clear cpuid data to avoid returning stale info. */ bzero(data->data, sizeof(data->data)); DPRINTF("[cpuctl,%d]: retriving cpuid level %#0x for %d cpu\n", __LINE__, data->level, cpu); oldcpu = td->td_oncpu; is_bound = cpu_sched_is_bound(td); set_cpu(cpu, td); cpuid_count(data->level, 0, data->data); restore_cpu(oldcpu, is_bound, td); return (0); }
void vmm_host_state_init(void) { int regs[4]; vmm_host_efer = rdmsr(MSR_EFER); vmm_host_pat = rdmsr(MSR_PAT); /* * We always want CR0.TS to be set when the processor does a VM exit. * * With emulation turned on unconditionally after a VM exit, we are * able to trap inadvertent use of the FPU until the guest FPU state * has been safely squirreled away. */ vmm_host_cr0 = rcr0() | CR0_TS; vmm_host_cr4 = rcr4(); /* * Only permit a guest to use XSAVE if the host is using * XSAVE. Only permit a guest to use XSAVE features supported * by the host. This ensures that the FPU state used by the * guest is always a subset of the saved guest FPU state. * * In addition, only permit known XSAVE features where the * rules for which features depend on other features is known * to properly emulate xsetbv. */ if (vmm_host_cr4 & CR4_XSAVE) { vmm_xsave_limits.xsave_enabled = 1; vmm_host_xcr0 = rxcr(0); vmm_xsave_limits.xcr0_allowed = vmm_host_xcr0 & (XFEATURE_AVX | XFEATURE_MPX | XFEATURE_AVX512); cpuid_count(0xd, 0x0, regs); vmm_xsave_limits.xsave_max_size = regs[1]; } }