static void ept_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start,
unsigned long end)
{
struct vmx_vcpu *vcpu = mmu_notifier_to_vmx(mn);
int ret;
epte_t *epte;
unsigned long pos = start;
bool sync_needed = false;
pr_debug("ept: invalidate_range_start start %lx end %lx\n", start, end);
spin_lock(&vcpu->ept_lock);
while (pos < end) {
ret = ept_lookup(vcpu, mm, (void *) pos, 0, 0, &epte);
if (!ret) {
pos += epte_big(*epte) ? HUGE_PAGE_SIZE : PAGE_SIZE;
ept_clear_epte(epte);
sync_needed = true;
} else
pos += PAGE_SIZE;
}
spin_unlock(&vcpu->ept_lock);
if (sync_needed)
vmx_ept_sync_vcpu(vcpu);
}
/**
* ept_invalidate_page - removes a page from the EPT
* @vcpu: the vcpu
* @mm: the process's mm_struct
* @addr: the address of the page
*
* Returns 1 if the page was removed, 0 otherwise
*/
static int ept_invalidate_page(struct vmx_vcpu *vcpu,
struct mm_struct *mm,
unsigned long addr)
{
int ret;
epte_t *epte;
void *gpa = (void *) hva_to_gpa(vcpu, mm, (unsigned long) addr);
if (gpa == (void *) GPA_ADDR_INVAL) {
printk(KERN_ERR "ept: hva %lx is out of range\n", addr);
return 0;
}
spin_lock(&vcpu->ept_lock);
ret = ept_lookup_gpa(vcpu, (void *) gpa, 0, 0, &epte);
if (ret) {
spin_unlock(&vcpu->ept_lock);
return 0;
}
ret = ept_clear_epte(epte);
spin_unlock(&vcpu->ept_lock);
if (ret)
vmx_ept_sync_individual_addr(vcpu, (gpa_t) gpa);
return ret;
}
static int ept_set_pfnmap_epte(struct vmx_vcpu *vcpu, int make_write,
unsigned long gpa, unsigned long hva)
{
struct vm_area_struct *vma;
struct mm_struct *mm = current->mm;
epte_t *epte, flags;
unsigned long pfn;
int ret;
down_read(&mm->mmap_sem);
vma = find_vma(mm, hva);
if (!vma) {
up_read(&mm->mmap_sem);
return -EFAULT;
}
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) {
up_read(&mm->mmap_sem);
return -EFAULT;
}
ret = follow_pfn(vma, hva, &pfn);
if (ret) {
up_read(&mm->mmap_sem);
return ret;
}
up_read(&mm->mmap_sem);
/* NOTE: pfn's can not be huge pages, which is quite a relief here */
spin_lock(&vcpu->ept_lock);
ret = ept_lookup_gpa(vcpu, (void *) gpa, 0, 1, &epte);
if (ret) {
spin_unlock(&vcpu->ept_lock);
printk(KERN_ERR "ept: failed to lookup EPT entry\n");
return ret;
}
flags = __EPTE_READ | __EPTE_TYPE(EPTE_TYPE_UC) |
__EPTE_IPAT | __EPTE_PFNMAP;
if (make_write)
flags |= __EPTE_WRITE;
if (vcpu->ept_ad_enabled) {
/* premark A/D to avoid extra memory references */
flags |= __EPTE_A;
if (make_write)
flags |= __EPTE_D;
}
if (epte_present(*epte))
ept_clear_epte(epte);
*epte = epte_addr(pfn << PAGE_SHIFT) | flags;
spin_unlock(&vcpu->ept_lock);
return 0;
}
static int ept_set_epte(struct vmx_vcpu *vcpu, int make_write,
unsigned long gpa, unsigned long hva)
{
int ret;
epte_t *epte, flags;
struct page *page;
ret = get_user_pages_fast(hva, 1, make_write, &page);
if (ret != 1) {
printk(KERN_ERR "ept: failed to get user page %lx\n", hva);
return ret;
}
spin_lock(&vcpu->ept_lock);
ret = ept_lookup_gpa(vcpu, (void *) gpa,
PageHuge(page) ? 1 : 0, 1, &epte);
if (ret) {
spin_unlock(&vcpu->ept_lock);
printk(KERN_ERR "ept: failed to lookup EPT entry\n");
return ret;
}
if (epte_present(*epte)) {
if (!epte_big(*epte) && PageHuge(page))
ept_clear_l1_epte(epte);
else
ept_clear_epte(epte);
}
flags = __EPTE_READ | __EPTE_EXEC |
__EPTE_TYPE(EPTE_TYPE_WB) | __EPTE_IPAT;
if (make_write)
flags |= __EPTE_WRITE;
if (vcpu->ept_ad_enabled) {
/* premark A/D to avoid extra memory references */
flags |= __EPTE_A;
if (make_write)
flags |= __EPTE_D;
}
if (PageHuge(page)) {
flags |= __EPTE_SZ;
*epte = epte_addr(page_to_phys(page) & ~((1 << 21) - 1)) |
flags;
} else
*epte = epte_addr(page_to_phys(page)) | flags;
spin_unlock(&vcpu->ept_lock);
return 0;
}
static int ept_set_epte(struct vmx_vcpu *vcpu, int make_write,
unsigned long gpa, unsigned long hva)
{
int ret;
epte_t *epte, flags;
struct page *page;
unsigned huge_shift;
int level;
ret = get_user_pages_fast(hva, 1, make_write, &page);
if (ret != 1) {
ret = ept_set_pfnmap_epte(vcpu, make_write, gpa, hva);
if (ret)
printk(KERN_ERR "ept: failed to get user page %lx\n", hva);
return ret;
}
spin_lock(&vcpu->ept_lock);
huge_shift = compound_order(compound_head(page)) + PAGE_SHIFT;
level = 0;
if (huge_shift == 30)
level = 2;
else if (huge_shift == 21)
level = 1;
ret = ept_lookup_gpa(vcpu, (void *) gpa,
level, 1, &epte);
if (ret) {
spin_unlock(&vcpu->ept_lock);
put_page(page);
printk(KERN_ERR "ept: failed to lookup EPT entry\n");
return ret;
}
if (epte_present(*epte)) {
if (!epte_big(*epte) && level == 2)
ept_clear_l2_epte(epte);
else if (!epte_big(*epte) && level == 1)
ept_clear_l1_epte(epte);
else
ept_clear_epte(epte);
}
flags = __EPTE_READ | __EPTE_EXEC |
__EPTE_TYPE(EPTE_TYPE_WB) | __EPTE_IPAT;
if (make_write)
flags |= __EPTE_WRITE;
if (vcpu->ept_ad_enabled) {
/* premark A/D to avoid extra memory references */
flags |= __EPTE_A;
if (make_write)
flags |= __EPTE_D;
}
if (level) {
struct page *tmp = page;
page = compound_head(page);
get_page(page);
put_page(tmp);
flags |= __EPTE_SZ;
}
*epte = epte_addr(page_to_phys(page)) | flags;
spin_unlock(&vcpu->ept_lock);
return 0;
}