/*
* Switch the owner of the FPU to the given thread.
*/
static void
switchFpuOwner(tcb_t *new_owner)
{
enableFpu();
if (ia32KSfpuOwner) {
saveFpuState(&ia32KSfpuOwner->tcbContext.fpuState);
}
if (new_owner) {
loadFpuState(&new_owner->tcbContext.fpuState);
} else {
disableFpu();
}
ia32KSfpuOwner = new_owner;
}
/*
* Initialise the FPU for this machine.
*/
BOOT_CODE bool_t
Arch_initFpu(void)
{
/* Enable FPU / SSE / SSE2 / SSE3 / SSSE3 / SSE4 Extensions. */
write_cr4(read_cr4() | CR4_OSFXSR);
/* Enable the FPU in general. */
write_cr0((read_cr0() & ~CR0_EMULATION) | CR0_MONITOR_COPROC | CR0_NUMERIC_ERROR);
enableFpu();
/* Initialize the fpu state */
finit();
if (config_set(CONFIG_XSAVE)) {
uint64_t xsave_features;
uint32_t xsave_instruction;
uint64_t desired_features = config_ternary(CONFIG_XSAVE, CONFIG_XSAVE_FEATURE_SET, 1);
xsave_state_t *nullFpuState = (xsave_state_t *) &x86KSnullFpuState;
/* create NULL state for FPU to be used by XSAVE variants */
memzero(&x86KSnullFpuState, sizeof(x86KSnullFpuState));
/* check for XSAVE support */
if (!(x86_cpuid_ecx(1, 0) & BIT(26))) {
printf("XSAVE not supported\n");
return false;
}
/* enable XSAVE support */
write_cr4(read_cr4() | CR4_OSXSAVE);
/* check feature mask */
xsave_features = ((uint64_t)x86_cpuid_edx(0x0d, 0x0) << 32) | x86_cpuid_eax(0x0d, 0x0);
if ((xsave_features & desired_features) != desired_features) {
printf("Requested feature mask is 0x%llx, but only 0x%llx supported\n", desired_features, (long long)xsave_features);
return false;
}
/* enable feature mask */
write_xcr0(desired_features);
/* validate the xsave buffer size and instruction */
if (x86_cpuid_ebx(0x0d, 0x0) > CONFIG_XSAVE_SIZE) {
printf("XSAVE buffer set set to %d, but needs to be at least %d\n", CONFIG_XSAVE_SIZE, x86_cpuid_ebx(0x0d, 0x0));
return false;
}
if (x86_cpuid_ebx(0x0d, 0x0) < CONFIG_XSAVE_SIZE) {
printf("XSAVE buffer set set to %d, but only needs to be %d.\n"
"Warning: Memory may be wasted with larger than needed TCBs.\n",
CONFIG_XSAVE_SIZE, x86_cpuid_ebx(0x0d, 0x0));
}
/* check if a specialized XSAVE instruction was requested */
xsave_instruction = x86_cpuid_eax(0x0d, 0x1);
if (config_set(CONFIG_XSAVE_XSAVEOPT)) {
if (!(xsave_instruction & BIT(0))) {
printf("XSAVEOPT requested, but not supported\n");
return false;
}
} else if (config_set(CONFIG_XSAVE_XSAVEC)) {
if (!(xsave_instruction & BIT(1))) {
printf("XSAVEC requested, but not supported\n");
return false;
}
} else if (config_set(CONFIG_XSAVE_XSAVES)) {
if (!(xsave_instruction & BIT(3))) {
printf("XSAVES requested, but not supported\n");
return false;
}
/* AVX state from extended region should be in compacted format */
nullFpuState->header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT;
/* initialize the XSS MSR */
x86_wrmsr(IA32_XSS_MSR, desired_features);
}
/* copy i387 FPU initial state from FPU */
saveFpuState(&x86KSnullFpuState);
nullFpuState->i387.mxcsr = MXCSR_INIT_VALUE;
} else {
/* Store the null fpu state */
saveFpuState(&x86KSnullFpuState);
}
/* Set the FPU to lazy switch mode */
disableFpu();
return true;
}
BOOT_CODE bool_t
init_sys_state(
cpu_id_t cpu_id,
mem_p_regs_t mem_p_regs,
dev_p_regs_t* dev_p_regs,
ui_info_t ui_info,
p_region_t boot_mem_reuse_p_reg,
/* parameters below not modeled in abstract specification */
uint32_t num_drhu,
paddr_t* drhu_list,
acpi_rmrr_list_t *rmrr_list
)
{
cap_t root_cnode_cap;
vptr_t bi_frame_vptr;
vptr_t ipcbuf_vptr;
cap_t it_vspace_cap;
cap_t it_ap_cap;
cap_t ipcbuf_cap;
pptr_t bi_frame_pptr;
create_frames_of_region_ret_t create_frames_ret;
#ifdef CONFIG_ENABLE_BENCHMARKS
vm_attributes_t buffer_attr = {{ 0 }};
word_t paddr;
pde_t pde;
#endif /* CONFIG_ENABLE_BENCHMARKS */
/* convert from physical addresses to kernel pptrs */
region_t ui_reg = paddr_to_pptr_reg(ui_info.p_reg);
region_t boot_mem_reuse_reg = paddr_to_pptr_reg(boot_mem_reuse_p_reg);
/* convert from physical addresses to userland vptrs */
v_region_t ui_v_reg;
v_region_t it_v_reg;
ui_v_reg.start = ui_info.p_reg.start - ui_info.pv_offset;
ui_v_reg.end = ui_info.p_reg.end - ui_info.pv_offset;
ipcbuf_vptr = ui_v_reg.end;
bi_frame_vptr = ipcbuf_vptr + BIT(PAGE_BITS);
/* The region of the initial thread is the user image + ipcbuf and boot info */
it_v_reg.start = ui_v_reg.start;
it_v_reg.end = bi_frame_vptr + BIT(PAGE_BITS);
init_freemem(ui_info.p_reg, mem_p_regs);
/* initialise virtual-memory-related data structures (not in abstract spec) */
if (!init_vm_state()) {
return false;
}
#ifdef CONFIG_ENABLE_BENCHMARKS
/* allocate and create the log buffer */
buffer_attr.words[0] = IA32_PAT_MT_WRITE_THROUGH;
paddr = pptr_to_paddr((void *) alloc_region(pageBitsForSize(X86_LargePage)));
/* allocate a large frame for logging */
pde = x86_make_pde_mapping(paddr, buffer_attr);
ia32KSGlobalPD[IA32_KSLOG_IDX] = pde;
/* flush the tlb */
invalidateTranslationAll();
/* if we crash here, the log isn't working */
#ifdef CONFIG_DEBUG_BUILD
#if CONFIG_MAX_NUM_TRACE_POINTS > 0
printf("Testing log\n");
ksLog[0].data = 0xdeadbeef;
printf("Wrote to ksLog %x\n", ksLog[0].data);
assert(ksLog[0].data == 0xdeadbeef);
#endif /* CONFIG_MAX_NUM_TRACE_POINTS */
#endif /* CONFIG_DEBUG_BUILD */
#endif /* CONFIG_ENABLE_BENCHMARKS */
/* create the root cnode */
root_cnode_cap = create_root_cnode();
/* create the IO port cap */
write_slot(
SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapIOPort),
cap_io_port_cap_new(
0, /* first port */
NUM_IO_PORTS - 1 /* last port */
)
);
/* create the cap for managing thread domains */
create_domain_cap(root_cnode_cap);
/* create the IRQ CNode */
if (!create_irq_cnode()) {
return false;
}
/* initialise the IRQ states and provide the IRQ control cap */
init_irqs(root_cnode_cap);
/* create the bootinfo frame */
bi_frame_pptr = allocate_bi_frame(0, 1, ipcbuf_vptr);
if (!bi_frame_pptr) {
return false;
}
/* Construct an initial address space with enough virtual addresses
* to cover the user image + ipc buffer and bootinfo frames */
it_vspace_cap = create_it_address_space(root_cnode_cap, it_v_reg);
if (cap_get_capType(it_vspace_cap) == cap_null_cap) {
return false;
}
/* Create and map bootinfo frame cap */
create_bi_frame_cap(
root_cnode_cap,
it_vspace_cap,
bi_frame_pptr,
bi_frame_vptr
);
/* create the initial thread's IPC buffer */
ipcbuf_cap = create_ipcbuf_frame(root_cnode_cap, it_vspace_cap, ipcbuf_vptr);
if (cap_get_capType(ipcbuf_cap) == cap_null_cap) {
return false;
}
/* create all userland image frames */
create_frames_ret =
create_frames_of_region(
root_cnode_cap,
it_vspace_cap,
ui_reg,
true,
ui_info.pv_offset
);
if (!create_frames_ret.success) {
return false;
}
ndks_boot.bi_frame->userImageFrames = create_frames_ret.region;
/* create the initial thread's ASID pool */
it_ap_cap = create_it_asid_pool(root_cnode_cap);
if (cap_get_capType(it_ap_cap) == cap_null_cap) {
return false;
}
write_it_asid_pool(it_ap_cap, it_vspace_cap);
/*
* Initialise the NULL FPU state. This is different from merely zero'ing it
* out (i.e., the NULL FPU state is non-zero), and must be performed before
* the first thread is created.
*/
resetFpu();
saveFpuState(&x86KSnullFpuState);
x86KSfpuOwner = NULL;
/* create the idle thread */
if (!create_idle_thread()) {
return false;
}
/* create the initial thread */
if (!create_initial_thread(
root_cnode_cap,
it_vspace_cap,
ui_info.v_entry,
bi_frame_vptr,
ipcbuf_vptr,
ipcbuf_cap
)) {
return false;
}
if (config_set(CONFIG_IOMMU)) {
/* initialise VTD-related data structures and the IOMMUs */
if (!vtd_init(cpu_id, num_drhu, rmrr_list)) {
return false;
}
/* write number of IOMMU PT levels into bootinfo */
ndks_boot.bi_frame->numIOPTLevels = x86KSnumIOPTLevels;
/* write IOSpace master cap */
write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapIOSpace), master_iospace_cap());
} else {
ndks_boot.bi_frame->numIOPTLevels = -1;
}
/* convert the remaining free memory into UT objects and provide the caps */
if (!create_untypeds(root_cnode_cap, boot_mem_reuse_reg)) {
return false;
}
/* WARNING: alloc_region() must not be called anymore after here! */
/* create device frames */
if (!create_device_frames(root_cnode_cap, dev_p_regs)) {
return false;
}
/* finalise the bootinfo frame */
bi_finalise();
return true;
}
static void
saveState (State *state) {
saveFpuState(&state->fpu);
getDiskTransferAddress(&state->dta);
state->psp = getProgramSegmentPrefix();
}
/*
* Initialise the FPU for this machine.
*/
BOOT_CODE bool_t
Arch_initFpu(void)
{
/* Enable FPU / SSE / SSE2 / SSE3 / SSSE3 / SSE4 Extensions. */
write_cr4(read_cr4() | CR4_OSFXSR);
/* Enable the FPU in general. */
write_cr0((read_cr0() & ~CR0_EMULATION) | CR0_MONITOR_COPROC | CR0_NUMERIC_ERROR);
enableFpu();
/* Initialize the fpu state */
finit();
if (config_set(CONFIG_XSAVE)) {
uint64_t xsave_features;
uint32_t xsave_instruction;
uint64_t desired_features = config_default(CONFIG_XSAVE_FEATURE_SET, 1);
/* check for XSAVE support */
if (!(x86_cpuid_ecx(1, 0) & BIT(26))) {
printf("XSAVE not supported\n");
return false;
}
/* enable XSAVE support */
write_cr4(read_cr4() | CR4_OSXSAVE);
/* check feature mask */
xsave_features = ((uint64_t)x86_cpuid_edx(0x0d, 0x0) << 32) | x86_cpuid_eax(0x0d, 0x0);
if ((xsave_features & desired_features) != desired_features) {
printf("Requested feature mask is 0x%llx, but only 0x%llx supported\n", desired_features, (long long)xsave_features);
return false;
}
/* enable feature mask */
write_xcr0(desired_features);
/* validate the xsave buffer size and instruction */
if (x86_cpuid_ebx(0x0d, 0x0) != CONFIG_XSAVE_SIZE) {
printf("XSAVE buffer set set to %d, but should be %d\n", CONFIG_XSAVE_SIZE, x86_cpuid_ecx(0x0d, 0x0));
return false;
}
/* check if a specialized XSAVE instruction was requested */
xsave_instruction = x86_cpuid_eax(0x0d, 0x1);
if (config_set(CONFIG_XSAVE_XSAVEOPT)) {
if (!(xsave_instruction & BIT(0))) {
printf("XSAVEOPT requested, but not supported\n");
return false;
}
} else if (config_set(CONFIG_XSAVE_XSAVEC)) {
if (!(xsave_instruction & BIT(1))) {
printf("XSAVEC requested, but not supported\n");
return false;
}
} else if (config_set(CONFIG_XSAVE_XSAVES)) {
if (!(xsave_instruction & BIT(3))) {
printf("XSAVES requested, but not supported\n");
return false;
}
/* initialize the XSS MSR */
x86_wrmsr(IA32_XSS_MSR, desired_features);
}
/* Load a NULL fpu state so that the idle thread ends up
* with a sensible FPU state and we can optimize our
* switch of it */
memzero(&x86KSnullFpuState, sizeof(x86KSnullFpuState));
loadFpuState(&x86KSnullFpuState);
} else {
/* Store the null fpu state */
saveFpuState(&x86KSnullFpuState);
}
/* Set the FPU to lazy switch mode */
disableFpu();
return true;
}
BOOT_CODE bool_t
init_node_state(
p_region_t avail_p_reg,
p_region_t sh_p_reg,
dev_p_regs_t* dev_p_regs,
ui_info_t ui_info,
p_region_t boot_mem_reuse_p_reg,
node_id_t node_id,
uint32_t num_nodes,
/* parameters below not modeled in abstract specification */
pdpte_t* kernel_pdpt,
pde_t* kernel_pd,
pte_t* kernel_pt
#ifdef CONFIG_IOMMU
, cpu_id_t cpu_id,
uint32_t num_drhu,
paddr_t* drhu_list,
uint32_t num_passthrough_dev,
dev_id_t* passthrough_dev_list,
uint32_t* pci_bus_used_bitmap
#endif
)
{
cap_t root_cnode_cap;
vptr_t bi_frame_vptr;
vptr_t ipcbuf_vptr;
cap_t it_vspace_cap;
cap_t it_ap_cap;
cap_t ipcbuf_cap;
pptr_t bi_frame_pptr;
create_frames_of_region_ret_t create_frames_ret;
int i;
#ifdef CONFIG_BENCHMARK
vm_attributes_t buffer_attr = {{ 0 }};
uint32_t paddr;
pde_t pde;
#endif /* CONFIG_BENCHMARK */
/* convert from physical addresses to kernel pptrs */
region_t avail_reg = paddr_to_pptr_reg(avail_p_reg);
region_t ui_reg = paddr_to_pptr_reg(ui_info.p_reg);
region_t sh_reg = paddr_to_pptr_reg(sh_p_reg);
region_t boot_mem_reuse_reg = paddr_to_pptr_reg(boot_mem_reuse_p_reg);
/* convert from physical addresses to userland vptrs */
v_region_t ui_v_reg;
v_region_t it_v_reg;
ui_v_reg.start = ui_info.p_reg.start - ui_info.pv_offset;
ui_v_reg.end = ui_info.p_reg.end - ui_info.pv_offset;
ipcbuf_vptr = ui_v_reg.end;
bi_frame_vptr = ipcbuf_vptr + BIT(PAGE_BITS);
/* The region of the initial thread is the user image + ipcbuf and boot info */
it_v_reg.start = ui_v_reg.start;
it_v_reg.end = bi_frame_vptr + BIT(PAGE_BITS);
/* make the free memory available to alloc_region() */
ndks_boot.freemem[0] = avail_reg;
for (i = 1; i < MAX_NUM_FREEMEM_REG; i++) {
ndks_boot.freemem[i] = REG_EMPTY;
}
/* initialise virtual-memory-related data structures (not in abstract spec) */
if (!init_vm_state(kernel_pdpt, kernel_pd, kernel_pt)) {
return false;
}
#ifdef CONFIG_BENCHMARK
/* allocate and create the log buffer */
buffer_attr.words[0] = IA32_PAT_MT_WRITE_THROUGH;
paddr = pptr_to_paddr((void *) alloc_region(pageBitsForSize(IA32_LargePage)));
/* allocate a large frame for logging */
pde = pde_pde_large_new(
paddr, /* page_base_address */
vm_attributes_get_ia32PATBit(buffer_attr), /* pat */
0, /* avl_cte_depth */
1, /* global */
0, /* dirty */
0, /* accessed */
vm_attributes_get_ia32PCDBit(buffer_attr), /* cache_disabled */
vm_attributes_get_ia32PWTBit(buffer_attr), /* write_through */
0, /* super_user */
1, /* read_write */
1 /* present */
);
/* TODO this shouldn't be hardcoded */
ia32KSkernelPD[IA32_KSLOG_IDX] = pde;
/* flush the tlb */
invalidatePageStructureCache();
/* if we crash here, the log isn't working */
#ifdef CONFIG_DEBUG_BUILD
printf("Testing log\n");
ksLog[0] = 0xdeadbeef;
printf("Wrote to ksLog %x\n", ksLog[0]);
assert(ksLog[0] == 0xdeadbeef);
#endif /* CONFIG_DEBUG_BUILD */
#endif /* CONFIG_BENCHMARK */
/* create the root cnode */
root_cnode_cap = create_root_cnode();
/* create the IO port cap */
write_slot(
SLOT_PTR(pptr_of_cap(root_cnode_cap), BI_CAP_IO_PORT),
cap_io_port_cap_new(
0, /* first port */
NUM_IO_PORTS - 1 /* last port */
)
);
/* create the cap for managing thread domains */
create_domain_cap(root_cnode_cap);
/* create the IRQ CNode */
if (!create_irq_cnode()) {
return false;
}
/* initialise the IRQ states and provide the IRQ control cap */
init_irqs(root_cnode_cap, node_id != 0);
/* create the bootinfo frame */
bi_frame_pptr = allocate_bi_frame(node_id, num_nodes, ipcbuf_vptr);
if (!bi_frame_pptr) {
return false;
}
/* Construct an initial address space with enough virtual addresses
* to cover the user image + ipc buffer and bootinfo frames */
it_vspace_cap = create_it_address_space(root_cnode_cap, it_v_reg);
if (cap_get_capType(it_vspace_cap) == cap_null_cap) {
return false;
}
/* Create and map bootinfo frame cap */
create_bi_frame_cap(
root_cnode_cap,
it_vspace_cap,
bi_frame_pptr,
bi_frame_vptr
);
/* create the initial thread's IPC buffer */
ipcbuf_cap = create_ipcbuf_frame(root_cnode_cap, it_vspace_cap, ipcbuf_vptr);
if (cap_get_capType(ipcbuf_cap) == cap_null_cap) {
return false;
}
/* create all userland image frames */
create_frames_ret =
create_frames_of_region(
root_cnode_cap,
it_vspace_cap,
ui_reg,
true,
ui_info.pv_offset
);
if (!create_frames_ret.success) {
return false;
}
ndks_boot.bi_frame->ui_frame_caps = create_frames_ret.region;
/* create the initial thread's ASID pool */
it_ap_cap = create_it_asid_pool(root_cnode_cap);
if (cap_get_capType(it_ap_cap) == cap_null_cap) {
return false;
}
write_it_asid_pool(it_ap_cap, it_vspace_cap);
/*
* Initialise the NULL FPU state. This is different from merely zero'ing it
* out (i.e., the NULL FPU state is non-zero), and must be performed before
* the first thread is created.
*/
resetFpu();
saveFpuState(&ia32KSnullFpuState);
ia32KSfpuOwner = NULL;
/* create the idle thread */
if (!create_idle_thread()) {
return false;
}
/* create the initial thread */
if (!create_initial_thread(
root_cnode_cap,
it_vspace_cap,
ui_info.v_entry,
bi_frame_vptr,
ipcbuf_vptr,
ipcbuf_cap
)) {
return false;
}
#ifdef CONFIG_IOMMU
/* initialise VTD-related data structures and the IOMMUs */
if (!vtd_init(cpu_id, num_drhu, pci_bus_used_bitmap, num_passthrough_dev, passthrough_dev_list)) {
return false;
}
/* write number of IOMMU PT levels into bootinfo */
ndks_boot.bi_frame->num_iopt_levels = ia32KSnumIOPTLevels;
/* write IOSpace master cap */
write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), BI_CAP_IO_SPACE), master_iospace_cap());
#endif
/* convert the remaining free memory into UT objects and provide the caps */
if (!create_untypeds(root_cnode_cap, boot_mem_reuse_reg)) {
return false;
}
/* WARNING: alloc_region() must not be called anymore after here! */
/* create device frames */
if (!create_device_frames(root_cnode_cap, dev_p_regs)) {
return false;
}
/* create all shared frames */
create_frames_ret =
create_frames_of_region(
root_cnode_cap,
it_vspace_cap,
sh_reg,
false,
0
);
if (!create_frames_ret.success) {
return false;
}
ndks_boot.bi_frame->sh_frame_caps = create_frames_ret.region;;
/* finalise the bootinfo frame */
bi_finalise();
#ifdef DEBUG
ia32KSconsolePort = console_port_of_node(node_id);
ia32KSdebugPort = debug_port_of_node(node_id);
#endif
return true;
}
static void saveState(struct State *state) {
saveFpuState(&state->fpu);
getDta(&state->dta);
state->psp = getPsp();
}
