int vm_do_unmap(addr_t virt, unsigned locked)
{
/* This gives the virtual address of the table needed, and sets
* the correct place as zero */
#if CONFIG_SWAP
if(current_task && num_swapdev && current_task->num_swapped)
swap_in_page((task_t *)current_task, virt & PAGE_MASK);
#endif
addr_t vpage = (virt&PAGE_MASK)/0x1000;
unsigned vp4 = PML4_IDX(vpage);
unsigned vpdpt = PDPT_IDX(vpage);
unsigned vdir = PAGE_DIR_IDX(vpage);
unsigned vtbl = PAGE_TABLE_IDX(vpage);
if(kernel_task && (virt&PAGE_MASK) != PDIR_DATA && !locked)
mutex_acquire(&pd_cur_data->lock);
page_dir_t *pd;
page_table_t *pt;
pdpt_t *pdpt;
pml4_t *pml4;
pml4 = (pml4_t *)((kernel_task && current_task) ? current_task->pd : kernel_dir);
if(!pml4[vp4])
pml4[vp4] = pm_alloc_page() | PAGE_PRESENT | PAGE_WRITE;
pdpt = (addr_t *)((pml4[vp4]&PAGE_MASK) + PHYS_PAGE_MAP);
if(!pdpt[vpdpt])
pdpt[vpdpt] = pm_alloc_page() | PAGE_PRESENT | PAGE_WRITE;
pd = (addr_t *)((pdpt[vpdpt]&PAGE_MASK) + PHYS_PAGE_MAP);
if(!pd[vdir])
pd[vdir] = pm_alloc_page() | PAGE_PRESENT | PAGE_WRITE;
pt = (addr_t *)((pd[vdir]&PAGE_MASK) + PHYS_PAGE_MAP);
addr_t p = pt[vtbl];
pt[vtbl] = 0;
asm("invlpg (%0)"::"r" (virt));
#if CONFIG_SMP
if(kernel_task && (virt&PAGE_MASK) != PDIR_DATA) {
if(IS_KERN_MEM(virt))
send_ipi(LAPIC_ICR_SHORT_OTHERS, 0, LAPIC_ICR_LEVELASSERT | LAPIC_ICR_TM_LEVEL | IPI_TLB);
else if((IS_THREAD_SHARED_MEM(virt) && pd_cur_data->count > 1))
send_ipi(LAPIC_ICR_SHORT_OTHERS, 0, LAPIC_ICR_LEVELASSERT | LAPIC_ICR_TM_LEVEL | IPI_TLB);
}
#endif
if(kernel_task && (virt&PAGE_MASK) != PDIR_DATA && !locked)
mutex_release(&pd_cur_data->lock);
if(p && !(p & PAGE_COW))
pm_free_page(p & PAGE_MASK);
return 0;
}
int vm_map(addr_t virt, addr_t phys, unsigned attr, unsigned opt)
{
addr_t vpage = (virt&PAGE_MASK)/0x1000;
unsigned vp4 = PML4_IDX(vpage);
unsigned vpdpt = PDPT_IDX(vpage);
unsigned vdir = PAGE_DIR_IDX(vpage);
unsigned vtbl = PAGE_TABLE_IDX(vpage);
if(kernel_task && !(opt & MAP_PDLOCKED))
mutex_acquire(&pd_cur_data->lock);
page_dir_t *pd;
page_table_t *pt;
pdpt_t *pdpt;
pml4_t *pml4;
pml4 = (pml4_t *)((kernel_task && current_task) ? current_task->pd : kernel_dir);
if(!pml4[vp4])
pml4[vp4] = pm_alloc_page_zero() | PAGE_PRESENT | PAGE_WRITE | (attr & PAGE_USER);
pdpt = (addr_t *)((pml4[vp4]&PAGE_MASK) + PHYS_PAGE_MAP);
if(!pdpt[vpdpt])
pdpt[vpdpt] = pm_alloc_page_zero() | PAGE_PRESENT | PAGE_WRITE | (attr & PAGE_USER);
pd = (addr_t *)((pdpt[vpdpt]&PAGE_MASK) + PHYS_PAGE_MAP);
if(!pd[vdir])
pd[vdir] = pm_alloc_page_zero() | PAGE_PRESENT | PAGE_WRITE | (attr & PAGE_USER);
pt = (addr_t *)((pd[vdir]&PAGE_MASK) + PHYS_PAGE_MAP);
pt[vtbl] = (phys & PAGE_MASK) | attr;
asm("invlpg (%0)"::"r" (virt));
if(!(opt & MAP_NOCLEAR))
memset((void *)(virt&PAGE_MASK), 0, 0x1000);
#if CONFIG_SMP
if(kernel_task) {
if(IS_KERN_MEM(virt))
send_ipi(LAPIC_ICR_SHORT_OTHERS, 0, LAPIC_ICR_LEVELASSERT | LAPIC_ICR_TM_LEVEL | IPI_TLB);
else if((IS_THREAD_SHARED_MEM(virt) && pd_cur_data->count > 1))
send_ipi(LAPIC_ICR_SHORT_OTHERS, 0, LAPIC_ICR_LEVELASSERT | LAPIC_ICR_TM_LEVEL | IPI_TLB);
}
#endif
if(kernel_task && !(opt & MAP_PDLOCKED))
mutex_release(&pd_cur_data->lock);
return 0;
}
pte_t *mmap_lookup(pml4e_t *pml4, uint64_t va, bool create)
{
struct page *page4pdp = NULL, *page4pd = NULL, *page4pt = NULL;
pdpe_t pml4e = pml4[PML4_IDX(va)];
if ((pml4e & PML4E_P) != 0)
goto pml4e_found;
if (create == false)
return NULL;
// Prepare new page directory pointer
if ((page4pdp = page_alloc()) == NULL)
return NULL;
memset(page2kva(page4pdp), 0, PAGE_SIZE);
page4pdp->ref = 1;
// Insert new pdp into PML4
pml4e = pml4[PML4_IDX(va)] = page2pa(page4pdp) | PML4E_P | PML4E_W | PML4E_U;
pml4e_found:
assert((pml4e & PML4E_P) != 0);
pdpe_t *pdp = VADDR(PML4E_ADDR(pml4e));
pdpe_t pdpe = pdp[PDP_IDX(va)];
if ((pdpe & PDPE_P) != 0)
goto pdpe_found;
if (create == false)
return NULL;
// Prepare new page directory
if ((page4pd = page_alloc()) == NULL)
return NULL;
memset(page2kva(page4pd), 0, PAGE_SIZE);
page4pd->ref = 1;
// Insert new page directory into page directory pointer table
pdpe = pdp[PDP_IDX(va)] = page2pa(page4pd) | PDPE_P | PDPE_W | PDPE_U;
pdpe_found:
assert((pdpe & PDPE_P) != 0);
pde_t *pd = VADDR(PDPE_ADDR(pdpe));
pde_t pde = pd[PD_IDX(va)];
if ((pde & PDE_P) != 0)
goto pde_found;
if (create == false)
return NULL;
// Prepare new page table
if ((page4pt = page_alloc()) == NULL)
return NULL;
memset(page2kva(page4pt), 0, PAGE_SIZE);
page4pt->ref = 1;
// Insert new page table into page directory
pde = pd[PD_IDX(va)] = page2pa(page4pt) | PDE_P | PTE_W | PDE_U;
pde_found:
assert((pde & PDE_P) != 0);
pte_t *pt = VADDR(PDE_ADDR(pde));
return &pt[PT_IDX(va)];
}
