static void xen_extend_mmu_update(const struct mmu_update *update) { struct multicall_space mcs; struct mmu_update *u; mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u)); if (mcs.mc != NULL) { ADD_STATS(mmu_update_extended, 1); ADD_STATS(mmu_update_histo[mcs.mc->args[1]], -1); mcs.mc->args[1]++; if (mcs.mc->args[1] < MMU_UPDATE_HISTO) ADD_STATS(mmu_update_histo[mcs.mc->args[1]], 1); else ADD_STATS(mmu_update_histo[0], 1); } else { ADD_STATS(mmu_update, 1); mcs = __xen_mc_entry(sizeof(*u)); MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); ADD_STATS(mmu_update_histo[1], 1); } u = mcs.args; *u = *update; }
/* This is called just after a mm has been created, but it has not been used yet. We need to make sure that its pagetable is all read-only, and can be pinned. */ void xen_pgd_pin(pgd_t *pgd) { struct multicall_space mcs; struct mmuext_op *op; xen_mc_batch(); if (pgd_walk(pgd, pin_page, TASK_SIZE)) { /* re-enable interrupts for kmap_flush_unused */ xen_mc_issue(0); kmap_flush_unused(); xen_mc_batch(); } mcs = __xen_mc_entry(sizeof(*op)); op = mcs.args; #ifdef CONFIG_X86_PAE op->cmd = MMUEXT_PIN_L3_TABLE; #else op->cmd = MMUEXT_PIN_L2_TABLE; #endif op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd))); MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); xen_mc_issue(0); }
static int pin_page(struct page *page, unsigned flags) { unsigned pgfl = test_and_set_bit(PG_pinned, &page->flags); int flush; if (pgfl) flush = 0; /* already pinned */ else if (PageHighMem(page)) /* kmaps need flushing if we found an unpinned highpage */ flush = 1; else { void *pt = lowmem_page_address(page); unsigned long pfn = page_to_pfn(page); struct multicall_space mcs = __xen_mc_entry(0); flush = 0; MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, pfn_pte(pfn, PAGE_KERNEL_RO), flags); } return flush; }
static void load_TLS_descriptor(struct thread_struct *t, unsigned int cpu, unsigned int i) { struct desc_struct *gdt = get_cpu_gdt_table(cpu); xmaddr_t maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]); struct multicall_space mc = __xen_mc_entry(0); MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]); }
static int unpin_page(struct page *page, unsigned flags) { unsigned pgfl = test_and_clear_bit(PG_pinned, &page->flags); if (pgfl && !PageHighMem(page)) { void *pt = lowmem_page_address(page); unsigned long pfn = page_to_pfn(page); struct multicall_space mcs = __xen_mc_entry(0); MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, pfn_pte(pfn, PAGE_KERNEL), flags); } return 0; /* never need to flush on unpin */ }
static void xen_extend_mmuext_op(const struct mmuext_op *op) { struct multicall_space mcs; struct mmuext_op *u; mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u)); if (mcs.mc != NULL) { mcs.mc->args[1]++; } else { mcs = __xen_mc_entry(sizeof(*u)); MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); } u = mcs.args; *u = *op; }
static void load_TLS_descriptor(struct thread_struct *t, unsigned int cpu, unsigned int i) { struct desc_struct *shadow = &per_cpu(shadow_tls_desc, cpu).desc[i]; struct desc_struct *gdt; xmaddr_t maddr; struct multicall_space mc; if (desc_equal(shadow, &t->tls_array[i])) return; *shadow = t->tls_array[i]; gdt = get_cpu_gdt_table(cpu); maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]); mc = __xen_mc_entry(0); MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]); }
/* Release a pagetables pages back as normal RW */ static void xen_pgd_unpin(pgd_t *pgd) { struct mmuext_op *op; struct multicall_space mcs; xen_mc_batch(); mcs = __xen_mc_entry(sizeof(*op)); op = mcs.args; op->cmd = MMUEXT_UNPIN_TABLE; op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd))); MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); pgd_walk(pgd, unpin_page, TASK_SIZE); xen_mc_issue(0); }
static int m2p_remove_override(struct page *page, struct gnttab_map_grant_ref *kmap_op, unsigned long mfn) { unsigned long flags; unsigned long pfn; unsigned long uninitialized_var(address); unsigned level; pte_t *ptep = NULL; pfn = page_to_pfn(page); if (!PageHighMem(page)) { address = (unsigned long)__va(pfn << PAGE_SHIFT); ptep = lookup_address(address, &level); if (WARN(ptep == NULL || level != PG_LEVEL_4K, "m2p_remove_override: pfn %lx not mapped", pfn)) return -EINVAL; } spin_lock_irqsave(&m2p_override_lock, flags); list_del(&page->lru); spin_unlock_irqrestore(&m2p_override_lock, flags); if (kmap_op != NULL) { if (!PageHighMem(page)) { struct multicall_space mcs; struct gnttab_unmap_and_replace *unmap_op; struct page *scratch_page = get_balloon_scratch_page(); unsigned long scratch_page_address = (unsigned long) __va(page_to_pfn(scratch_page) << PAGE_SHIFT); /* * It might be that we queued all the m2p grant table * hypercalls in a multicall, then m2p_remove_override * get called before the multicall has actually been * issued. In this case handle is going to -1 because * it hasn't been modified yet. */ if (kmap_op->handle == -1) xen_mc_flush(); /* * Now if kmap_op->handle is negative it means that the * hypercall actually returned an error. */ if (kmap_op->handle == GNTST_general_error) { pr_warn("m2p_remove_override: pfn %lx mfn %lx, failed to modify kernel mappings", pfn, mfn); put_balloon_scratch_page(); return -1; } xen_mc_batch(); mcs = __xen_mc_entry( sizeof(struct gnttab_unmap_and_replace)); unmap_op = mcs.args; unmap_op->host_addr = kmap_op->host_addr; unmap_op->new_addr = scratch_page_address; unmap_op->handle = kmap_op->handle; MULTI_grant_table_op(mcs.mc, GNTTABOP_unmap_and_replace, unmap_op, 1); mcs = __xen_mc_entry(0); MULTI_update_va_mapping(mcs.mc, scratch_page_address, pfn_pte(page_to_pfn(scratch_page), PAGE_KERNEL_RO), 0); xen_mc_issue(PARAVIRT_LAZY_MMU); kmap_op->host_addr = 0; put_balloon_scratch_page(); } } /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present * somewhere in this domain, even before being added to the * m2p_override (see comment above in m2p_add_override). * If there are no other entries in the m2p_override corresponding * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for * the original pfn (the one shared by the frontend): the backend * cannot do any IO on this page anymore because it has been * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of * the original pfn causes mfn_to_pfn(mfn) to return the frontend * pfn again. */ mfn &= ~FOREIGN_FRAME_BIT; pfn = mfn_to_pfn_no_overrides(mfn); if (__pfn_to_mfn(pfn) == FOREIGN_FRAME(mfn) && m2p_find_override(mfn) == NULL) set_phys_to_machine(pfn, mfn); return 0; }