static inline pool_pg_desc_t* pool_get_desc(offset_t pg)
{
size_t mn, mx, nr;
nr = page_nr(pg);
mn = page_nr(info->vmm.pool.addr);
mx = page_nr(info->vmm.pool.addr + info->vmm.pool.sz);
if(nr < mn || nr >= mx)
return (pool_pg_desc_t*)0;
return &info->vmm.pool.all[nr-mn];
}
/*
** we map more than available
** since we only use 1GB or 2MB pages
*/
static void vmm_pagemem_init()
{
pml4e_t *pml4;
pdpe_t *pdp;
pde64_t *pd;
cr3_reg_t cr3;
offset_t pfn;
uint32_t i, j, k;
offset_t limit;
size_t pdp_nr, pd_nr, pt_nr;
size_t pml4e_max, pdpe_max, pde_max;
pml4 = info->vmm.cpu.pg.pml4;
limit = info->hrd.mem.top - 1;
pdp_nr = pdp_nr(limit) + 1;
pd_nr = pd64_nr(limit) + 1;
pt_nr = pt64_nr(limit) + 1;
pfn = 0;
pml4e_max = pdp_nr; /* only one pml4 */
for(i=0 ; i<pml4e_max ; i++)
{
pdp = info->vmm.cpu.pg.pdp[i];
pg_set_entry(&pml4[i], PG_KRN|PG_RW, page_nr(pdp));
pdpe_max = min(pd_nr, PDPE_PER_PDP);
pd_nr -= pdpe_max;
for(j=0 ; j<pdpe_max ; j++)
{
if(info->vmm.cpu.skillz.pg_1G)
pg_set_large_entry(&pdp[j], PG_KRN|PG_RW, pfn++);
else
{
pd = info->vmm.cpu.pg.pd[j];
pg_set_entry(&pdp[j], PG_KRN|PG_RW, page_nr(pd));
pde_max = min(pt_nr, PDE64_PER_PD);
pt_nr -= pde_max;
for(k=0 ; k<pde_max ; k++)
pg_set_large_entry(&pd[k], PG_KRN|PG_RW, pfn++);
}
}
}
cr3.raw = 0UL;
cr3.pml4.addr = page_nr(pml4);
set_cr3(cr3.raw);
}
static void vm_pagemem_init()
{
npg_pdpe_t *pdp = info->vm.cpu.pg[0].pdp[0];
npg_init();
npg_set_entry(&info->vm.cpu.pg[0].pml4[0], npg_dft_attr, page_nr(pdp));
}
static struct page *alloc_new_pde(void)
{
struct page *page;
size_t size = PDE_TABLE_SIZE;
u32 align = PDE_TABLE_ALIGN_SIZE;
spin_lock_irqsave(&pgt_buffer.pgt_lock);
if (pgt_buffer.nr_free == 0) {
/*
* alloc new pgt from system
*/
page = get_free_pages_align(page_nr(size), align, GFP_PGT);
if (!page) {
spin_unlock_irqstore(&pgt_buffer.pgt_lock);
return NULL;
}
pgt_buffer.alloc_nr++;
} else {
page = list_to_page(list_next(&pgt_buffer.list));
list_del(list_next(&pgt_buffer.list));
pgt_buffer.nr_free--;
}
spin_unlock_irqstore(&pgt_buffer.pgt_lock);
return page;
}
ppg_dsc_t* ppg_get_desc(offset_t addr)
{
ppg_info_t *ppg = &info->hrd.mem.ppg;
offset_t n = page_nr(addr);
if(n < ppg->nr)
return &ppg->dsc[n];
return ((ppg_dsc_t*)0);
}
static void _npg_map_4K(offset_t addr, uint64_t attr)
{
npg_pdpe_t *pdpe = __npg_resolve_pdpe(addr, attr);
npg_pde64_t *pde = __npg_resolve_pde(pdpe, addr, attr);
npg_pte64_t *pte;
debug(PG_MAP, "map 4K 0x%X 0x%X\n", addr, attr);
if(npg_large(pdpe) || npg_large(pde))
return;
pte = __npg_resolve_pte(pde, addr, attr);
npg_set_page_entry(pte, attr, page_nr(addr));
}
static void __npg_map_1G_nolarge(npg_pdpe_t *pdpe, offset_t addr, uint64_t attr)
{
npg_pde64_t *pd;
uint32_t i;
pd = __npg_new_pd();
debug(PG_MAP, "map 2M for each pde\n");
for(i=0 ; i<PDE64_PER_PD ; i++)
{
__npg_map_2M(&pd[i], addr, attr);
addr += PG_2M_SIZE;
}
/* upper-level entry has full pvl */
npg_set_entry(pdpe, attr|npg_dft_pvl, page_nr(pd));
debug(PG_MAP, "mapped 1G until 0x%X 0x%X\n", addr, attr);
}
/*
** Low level services
*/
static void __npg_map_2M_nolarge(npg_pde64_t *pde, offset_t addr, uint64_t attr)
{
npg_pte64_t *pt;
offset_t pfn;
uint32_t i;
pt = __npg_new_pt();
debug(PG_MAP, "new pt\n");
pfn = pg_4K_nr(addr);
debug(PG_MAP, "mapping 4K for each pte\n");
for(i=0 ; i<PTE64_PER_PT ; i++, pfn++)
npg_set_page_entry(&pt[i], attr, pfn);
/* upper-level entry has full pvl */
npg_set_entry(pde, attr|npg_dft_pvl, page_nr(pt));
debug(PG_MAP, "mapped 2M 0x%X 0x%X\n", addr, attr);
}
/*
** resolve pte (allocate pt if needed for non-large pde
** return 0 if large page
*/
static inline
npg_pte64_t* __npg_resolve_pte(npg_pde64_t *pde, offset_t addr, uint64_t attr)
{
npg_pte64_t *pt;
if(!npg_present(pde))
{
pt = __npg_new_pt();
/* upper-level entry has full pvl */
npg_set_entry(pde, attr|npg_dft_pvl, page_nr(pt));
}
else if(npg_large(pde))
return 0;
else
pt = (npg_pte64_t*)page_addr(pde->addr);
return &pt[pt64_idx(addr)];
}
/*
** resolve pde (allocate pd if needed) for non-large pdpe
** return 0 if large page
*/
static inline
npg_pde64_t* __npg_resolve_pde(npg_pdpe_t *pdpe, offset_t addr, uint64_t attr)
{
npg_pde64_t *pd;
if(!npg_present(pdpe))
{
pd = __npg_new_pd();
/* upper-level entry has full pvl */
npg_set_entry(pdpe, attr|npg_dft_pvl, page_nr(pd));
}
else if(npg_large(pdpe))
return 0;
else
pd = (npg_pde64_t*)page_addr(pdpe->addr);
return &pd[pd64_idx(addr)];
}
/*
** resolve pdpe (allocate pdp if needed)
*/
static inline npg_pdpe_t* __npg_resolve_pdpe(offset_t addr, uint64_t attr)
{
vm_pgmem_t *pg = npg_get_active_paging();
npg_pml4e_t *pml4e;
npg_pdpe_t *pdp;
pml4e = &pg->pml4[pml4_idx(addr)];
if(!npg_present(pml4e))
{
pdp = __npg_new_pdp();
/* upper-level entry has full pvl */
npg_set_entry(pml4e, attr|npg_dft_pvl, page_nr(pdp));
}
else
pdp = (npg_pdpe_t*)page_addr(pml4e->addr);
return &pdp[pdp_idx(addr)];
}
int __init_text pgt_init(void)
{
size_t size;
int i; u32 align;
struct page *page;
int count = CONFIG_PGT_PDE_BUFFER_MIN;
/* this function needed called after mmu is init */
memset((char *)&pgt_buffer, 0, sizeof(struct pgt_buffer));
size = PDE_TABLE_SIZE;
align = PDE_TABLE_ALIGN_SIZE;
init_list(&pgt_buffer.list);
spin_lock_init(&pgt_buffer.pgt_lock);
/* do not alloc buffer if align size is PAGE_SIZE */
if (align == PAGE_SIZE)
count = 0;
/* Boot pde need to be care about */
page = va_to_page(mmu_get_boot_pde());
if (page) {
add_page_to_list_tail(page, &pgt_buffer.list);
pgt_buffer.alloc_nr++;
pgt_buffer.nr_free++;
}
for (i = 0; i < count; i++) {
page = get_free_pages_align(page_nr(size),
align, GFP_PGT);
if (!page)
break;
add_page_to_list_tail(page, &pgt_buffer.list);
pgt_buffer.alloc_nr++;
pgt_buffer.nr_free++;
}
return 0;
}
static void svm_vmcb_controls_init()
{
amd_svm_feat_t svm_feat;
cpuid_amd_svm_features(svm_feat);
if(svm_feat.asid_nr < 1)
panic("no asid");
if(!svm_feat.lbr)
panic("no lbr virtualization");
vm_ctrls.lbr.raw = 1ULL;
vm_ctrls.exception_bitmap.raw = info->vm.cpu.dflt_excp;
vm_ctrls.cr_read_bitmap = VMCB_CR_R_BITMAP;
vm_ctrls.cr_write_bitmap = VMCB_CR_W_BITMAP;
vm_ctrls.sys_insn_bitmap.cr0_sel_write = 1;
vm_ctrls.sys_insn_bitmap.cpuid = 1;
vm_ctrls.sys_insn_bitmap.task_sw = 1;
vm_ctrls.sys_insn_bitmap.freez = 1;
vm_ctrls.sys_insn_bitmap.shutdown = 1;
vm_ctrls.vmm_insn_bitmap.vmrun = 1;
vm_ctrls.vmm_insn_bitmap.vmmcall = 1;
vm_ctrls.vmm_insn_bitmap.vmload = 1;
vm_ctrls.vmm_insn_bitmap.vmsave = 1;
vm_ctrls.vmm_insn_bitmap.skinit = 1;
vm_ctrls.vmm_insn_bitmap.rdtscp = 1;
vm_ctrls.vmm_insn_bitmap.icebp = 1;
vm_ctrls.vmm_insn_bitmap.monitor = 1;
vm_ctrls.vmm_insn_bitmap.mwait = 1;
vm_ctrls.vmm_insn_bitmap.mwait_cond = 1;
info->vm.asid_nr = svm_feat.asid_nr;
vm_ctrls.sys_insn_bitmap.invlpga = 1;
vm_ctrls.tlb_ctrl.guest_asid = 1;
vm_ctrls.npt.raw = 1UL;
vm_ctrls.ncr3.pml4.addr = page_nr(info->vm.cpu.pg[0].pml4);
vm_ctrls.sys_insn_bitmap.intn = 1;
vm_ctrls.vmm_insn_bitmap.stgi = 1;
vm_ctrls.vmm_insn_bitmap.clgi = 1;
vm_ctrls.sys_insn_bitmap.init = 1;
vm_ctrls.sys_insn_bitmap.nmi = 1;
vm_ctrls.int_ctrl.v_ign_tpr = 1;
/*
** XXX: SMI not intercepted may disable interrupts
** (cf. revision guide for 10h familly)
** AMD tell to launch SMM in a guest
** to safely process i/o smi
** However, bios lock SMM regs
** and we are not able to properly
** handle all smi cases
**
** So we don't intercept !
*/
}
