static inline pool_pg_desc_t* pool_get_desc(offset_t pg) { size_t mn, mx, nr; nr = page_nr(pg); mn = page_nr(info->vmm.pool.addr); mx = page_nr(info->vmm.pool.addr + info->vmm.pool.sz); if(nr < mn || nr >= mx) return (pool_pg_desc_t*)0; return &info->vmm.pool.all[nr-mn]; }
/* ** we map more than available ** since we only use 1GB or 2MB pages */ static void vmm_pagemem_init() { pml4e_t *pml4; pdpe_t *pdp; pde64_t *pd; cr3_reg_t cr3; offset_t pfn; uint32_t i, j, k; offset_t limit; size_t pdp_nr, pd_nr, pt_nr; size_t pml4e_max, pdpe_max, pde_max; pml4 = info->vmm.cpu.pg.pml4; limit = info->hrd.mem.top - 1; pdp_nr = pdp_nr(limit) + 1; pd_nr = pd64_nr(limit) + 1; pt_nr = pt64_nr(limit) + 1; pfn = 0; pml4e_max = pdp_nr; /* only one pml4 */ for(i=0 ; i<pml4e_max ; i++) { pdp = info->vmm.cpu.pg.pdp[i]; pg_set_entry(&pml4[i], PG_KRN|PG_RW, page_nr(pdp)); pdpe_max = min(pd_nr, PDPE_PER_PDP); pd_nr -= pdpe_max; for(j=0 ; j<pdpe_max ; j++) { if(info->vmm.cpu.skillz.pg_1G) pg_set_large_entry(&pdp[j], PG_KRN|PG_RW, pfn++); else { pd = info->vmm.cpu.pg.pd[j]; pg_set_entry(&pdp[j], PG_KRN|PG_RW, page_nr(pd)); pde_max = min(pt_nr, PDE64_PER_PD); pt_nr -= pde_max; for(k=0 ; k<pde_max ; k++) pg_set_large_entry(&pd[k], PG_KRN|PG_RW, pfn++); } } } cr3.raw = 0UL; cr3.pml4.addr = page_nr(pml4); set_cr3(cr3.raw); }
static void vm_pagemem_init() { npg_pdpe_t *pdp = info->vm.cpu.pg[0].pdp[0]; npg_init(); npg_set_entry(&info->vm.cpu.pg[0].pml4[0], npg_dft_attr, page_nr(pdp)); }
static struct page *alloc_new_pde(void) { struct page *page; size_t size = PDE_TABLE_SIZE; u32 align = PDE_TABLE_ALIGN_SIZE; spin_lock_irqsave(&pgt_buffer.pgt_lock); if (pgt_buffer.nr_free == 0) { /* * alloc new pgt from system */ page = get_free_pages_align(page_nr(size), align, GFP_PGT); if (!page) { spin_unlock_irqstore(&pgt_buffer.pgt_lock); return NULL; } pgt_buffer.alloc_nr++; } else { page = list_to_page(list_next(&pgt_buffer.list)); list_del(list_next(&pgt_buffer.list)); pgt_buffer.nr_free--; } spin_unlock_irqstore(&pgt_buffer.pgt_lock); return page; }
ppg_dsc_t* ppg_get_desc(offset_t addr) { ppg_info_t *ppg = &info->hrd.mem.ppg; offset_t n = page_nr(addr); if(n < ppg->nr) return &ppg->dsc[n]; return ((ppg_dsc_t*)0); }
static void _npg_map_4K(offset_t addr, uint64_t attr) { npg_pdpe_t *pdpe = __npg_resolve_pdpe(addr, attr); npg_pde64_t *pde = __npg_resolve_pde(pdpe, addr, attr); npg_pte64_t *pte; debug(PG_MAP, "map 4K 0x%X 0x%X\n", addr, attr); if(npg_large(pdpe) || npg_large(pde)) return; pte = __npg_resolve_pte(pde, addr, attr); npg_set_page_entry(pte, attr, page_nr(addr)); }
static void __npg_map_1G_nolarge(npg_pdpe_t *pdpe, offset_t addr, uint64_t attr) { npg_pde64_t *pd; uint32_t i; pd = __npg_new_pd(); debug(PG_MAP, "map 2M for each pde\n"); for(i=0 ; i<PDE64_PER_PD ; i++) { __npg_map_2M(&pd[i], addr, attr); addr += PG_2M_SIZE; } /* upper-level entry has full pvl */ npg_set_entry(pdpe, attr|npg_dft_pvl, page_nr(pd)); debug(PG_MAP, "mapped 1G until 0x%X 0x%X\n", addr, attr); }
/* ** Low level services */ static void __npg_map_2M_nolarge(npg_pde64_t *pde, offset_t addr, uint64_t attr) { npg_pte64_t *pt; offset_t pfn; uint32_t i; pt = __npg_new_pt(); debug(PG_MAP, "new pt\n"); pfn = pg_4K_nr(addr); debug(PG_MAP, "mapping 4K for each pte\n"); for(i=0 ; i<PTE64_PER_PT ; i++, pfn++) npg_set_page_entry(&pt[i], attr, pfn); /* upper-level entry has full pvl */ npg_set_entry(pde, attr|npg_dft_pvl, page_nr(pt)); debug(PG_MAP, "mapped 2M 0x%X 0x%X\n", addr, attr); }
/* ** resolve pte (allocate pt if needed for non-large pde ** return 0 if large page */ static inline npg_pte64_t* __npg_resolve_pte(npg_pde64_t *pde, offset_t addr, uint64_t attr) { npg_pte64_t *pt; if(!npg_present(pde)) { pt = __npg_new_pt(); /* upper-level entry has full pvl */ npg_set_entry(pde, attr|npg_dft_pvl, page_nr(pt)); } else if(npg_large(pde)) return 0; else pt = (npg_pte64_t*)page_addr(pde->addr); return &pt[pt64_idx(addr)]; }
/* ** resolve pde (allocate pd if needed) for non-large pdpe ** return 0 if large page */ static inline npg_pde64_t* __npg_resolve_pde(npg_pdpe_t *pdpe, offset_t addr, uint64_t attr) { npg_pde64_t *pd; if(!npg_present(pdpe)) { pd = __npg_new_pd(); /* upper-level entry has full pvl */ npg_set_entry(pdpe, attr|npg_dft_pvl, page_nr(pd)); } else if(npg_large(pdpe)) return 0; else pd = (npg_pde64_t*)page_addr(pdpe->addr); return &pd[pd64_idx(addr)]; }
/* ** resolve pdpe (allocate pdp if needed) */ static inline npg_pdpe_t* __npg_resolve_pdpe(offset_t addr, uint64_t attr) { vm_pgmem_t *pg = npg_get_active_paging(); npg_pml4e_t *pml4e; npg_pdpe_t *pdp; pml4e = &pg->pml4[pml4_idx(addr)]; if(!npg_present(pml4e)) { pdp = __npg_new_pdp(); /* upper-level entry has full pvl */ npg_set_entry(pml4e, attr|npg_dft_pvl, page_nr(pdp)); } else pdp = (npg_pdpe_t*)page_addr(pml4e->addr); return &pdp[pdp_idx(addr)]; }
int __init_text pgt_init(void) { size_t size; int i; u32 align; struct page *page; int count = CONFIG_PGT_PDE_BUFFER_MIN; /* this function needed called after mmu is init */ memset((char *)&pgt_buffer, 0, sizeof(struct pgt_buffer)); size = PDE_TABLE_SIZE; align = PDE_TABLE_ALIGN_SIZE; init_list(&pgt_buffer.list); spin_lock_init(&pgt_buffer.pgt_lock); /* do not alloc buffer if align size is PAGE_SIZE */ if (align == PAGE_SIZE) count = 0; /* Boot pde need to be care about */ page = va_to_page(mmu_get_boot_pde()); if (page) { add_page_to_list_tail(page, &pgt_buffer.list); pgt_buffer.alloc_nr++; pgt_buffer.nr_free++; } for (i = 0; i < count; i++) { page = get_free_pages_align(page_nr(size), align, GFP_PGT); if (!page) break; add_page_to_list_tail(page, &pgt_buffer.list); pgt_buffer.alloc_nr++; pgt_buffer.nr_free++; } return 0; }
static void svm_vmcb_controls_init() { amd_svm_feat_t svm_feat; cpuid_amd_svm_features(svm_feat); if(svm_feat.asid_nr < 1) panic("no asid"); if(!svm_feat.lbr) panic("no lbr virtualization"); vm_ctrls.lbr.raw = 1ULL; vm_ctrls.exception_bitmap.raw = info->vm.cpu.dflt_excp; vm_ctrls.cr_read_bitmap = VMCB_CR_R_BITMAP; vm_ctrls.cr_write_bitmap = VMCB_CR_W_BITMAP; vm_ctrls.sys_insn_bitmap.cr0_sel_write = 1; vm_ctrls.sys_insn_bitmap.cpuid = 1; vm_ctrls.sys_insn_bitmap.task_sw = 1; vm_ctrls.sys_insn_bitmap.freez = 1; vm_ctrls.sys_insn_bitmap.shutdown = 1; vm_ctrls.vmm_insn_bitmap.vmrun = 1; vm_ctrls.vmm_insn_bitmap.vmmcall = 1; vm_ctrls.vmm_insn_bitmap.vmload = 1; vm_ctrls.vmm_insn_bitmap.vmsave = 1; vm_ctrls.vmm_insn_bitmap.skinit = 1; vm_ctrls.vmm_insn_bitmap.rdtscp = 1; vm_ctrls.vmm_insn_bitmap.icebp = 1; vm_ctrls.vmm_insn_bitmap.monitor = 1; vm_ctrls.vmm_insn_bitmap.mwait = 1; vm_ctrls.vmm_insn_bitmap.mwait_cond = 1; info->vm.asid_nr = svm_feat.asid_nr; vm_ctrls.sys_insn_bitmap.invlpga = 1; vm_ctrls.tlb_ctrl.guest_asid = 1; vm_ctrls.npt.raw = 1UL; vm_ctrls.ncr3.pml4.addr = page_nr(info->vm.cpu.pg[0].pml4); vm_ctrls.sys_insn_bitmap.intn = 1; vm_ctrls.vmm_insn_bitmap.stgi = 1; vm_ctrls.vmm_insn_bitmap.clgi = 1; vm_ctrls.sys_insn_bitmap.init = 1; vm_ctrls.sys_insn_bitmap.nmi = 1; vm_ctrls.int_ctrl.v_ign_tpr = 1; /* ** XXX: SMI not intercepted may disable interrupts ** (cf. revision guide for 10h familly) ** AMD tell to launch SMM in a guest ** to safely process i/o smi ** However, bios lock SMM regs ** and we are not able to properly ** handle all smi cases ** ** So we don't intercept ! */ }