/* Load a new IDT into Xen. In principle this can be per-CPU, so we hold a spinlock to protect the static traps[] array (static because it avoids allocation, and saves stack space). */ static void xen_load_idt(const struct desc_ptr *desc) { static DEFINE_SPINLOCK(lock); static struct trap_info traps[257]; trace_xen_cpu_load_idt(desc); spin_lock(&lock); __get_cpu_var(idt_desc) = *desc; xen_convert_trap_info(desc, traps); xen_mc_flush(); if (HYPERVISOR_set_trap_table(traps)) BUG(); spin_unlock(&lock); }
struct multicall_space __xen_mc_entry(size_t args) { struct mc_buffer *b = &__get_cpu_var(mc_buffer); struct multicall_space ret; unsigned argspace = (args + sizeof(u64) - 1) / sizeof(u64); BUG_ON(preemptible()); BUG_ON(argspace > MC_ARGS); if (b->mcidx == MC_BATCH || (b->argidx + argspace) > MC_ARGS) xen_mc_flush(); ret.mc = &b->entries[b->mcidx]; b->mcidx++; ret.args = &b->args[b->argidx]; b->argidx += argspace; return ret; }
/* Write a GDT descriptor entry. Ignore LDT descriptors, since they're handled differently. */ static void xen_write_gdt_entry(struct desc_struct *dt, int entry, const void *desc, int type) { preempt_disable(); switch (type) { case DESC_LDT: case DESC_TSS: /* ignore */ break; default: { xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]); xen_mc_flush(); if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc)) BUG(); } } preempt_enable(); }
struct multicall_space __xen_mc_entry(size_t args) { struct mc_buffer *b = &__get_cpu_var(mc_buffer); struct multicall_space ret; unsigned argidx = roundup(b->argidx, sizeof(u64)); BUG_ON(preemptible()); BUG_ON(b->argidx > MC_ARGS); if (b->mcidx == MC_BATCH || (argidx + args) > MC_ARGS) { mc_stats_flush(b->mcidx == MC_BATCH ? FL_SLOTS : FL_ARGS); xen_mc_flush(); argidx = roundup(b->argidx, sizeof(u64)); } ret.mc = &b->entries[b->mcidx]; b->mcidx++; ret.args = &b->args[argidx]; b->argidx = argidx + args; BUG_ON(b->argidx > MC_ARGS); return ret; }
static void xen_end_context_switch(struct task_struct *next) { xen_mc_flush(); paravirt_end_context_switch(next); }
int m2p_remove_override(struct page *page, bool clear_pte) { unsigned long flags; unsigned long mfn; unsigned long pfn; unsigned long uninitialized_var(address); unsigned level; pte_t *ptep = NULL; int ret = 0; pfn = page_to_pfn(page); mfn = get_phys_to_machine(pfn); if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT)) return -EINVAL; if (!PageHighMem(page)) { address = (unsigned long)__va(pfn << PAGE_SHIFT); ptep = lookup_address(address, &level); if (WARN(ptep == NULL || level != PG_LEVEL_4K, "m2p_remove_override: pfn %lx not mapped", pfn)) return -EINVAL; } spin_lock_irqsave(&m2p_override_lock, flags); list_del(&page->lru); spin_unlock_irqrestore(&m2p_override_lock, flags); WARN_ON(!PagePrivate(page)); ClearPagePrivate(page); if (clear_pte) { struct gnttab_map_grant_ref *map_op = (struct gnttab_map_grant_ref *) page->index; set_phys_to_machine(pfn, map_op->dev_bus_addr); if (!PageHighMem(page)) { struct multicall_space mcs; struct gnttab_unmap_grant_ref *unmap_op; /* * It might be that we queued all the m2p grant table * hypercalls in a multicall, then m2p_remove_override * get called before the multicall has actually been * issued. In this case handle is going to -1 because * it hasn't been modified yet. */ if (map_op->handle == -1) xen_mc_flush(); /* * Now if map_op->handle is negative it means that the * hypercall actually returned an error. */ if (map_op->handle == GNTST_general_error) { printk(KERN_WARNING "m2p_remove_override: " "pfn %lx mfn %lx, failed to modify kernel mappings", pfn, mfn); return -1; } mcs = xen_mc_entry( sizeof(struct gnttab_unmap_grant_ref)); unmap_op = mcs.args; unmap_op->host_addr = map_op->host_addr; unmap_op->handle = map_op->handle; unmap_op->dev_bus_addr = 0; MULTI_grant_table_op(mcs.mc, GNTTABOP_unmap_grant_ref, unmap_op, 1); xen_mc_issue(PARAVIRT_LAZY_MMU); set_pte_at(&init_mm, address, ptep, pfn_pte(pfn, PAGE_KERNEL)); __flush_tlb_single(address); map_op->host_addr = 0; } } else set_phys_to_machine(pfn, page->index); /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present * somewhere in this domain, even before being added to the * m2p_override (see comment above in m2p_add_override). * If there are no other entries in the m2p_override corresponding * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for * the original pfn (the one shared by the frontend): the backend * cannot do any IO on this page anymore because it has been * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of * the original pfn causes mfn_to_pfn(mfn) to return the frontend * pfn again. */ mfn &= ~FOREIGN_FRAME_BIT; ret = __get_user(pfn, &machine_to_phys_mapping[mfn]); if (ret == 0 && get_phys_to_machine(pfn) == FOREIGN_FRAME(mfn) && m2p_find_override(mfn) == NULL) set_phys_to_machine(pfn, mfn); return 0; }
void xen_leave_lazy(void) { paravirt_leave_lazy(paravirt_get_lazy_mode()); xen_mc_flush(); }
static int m2p_remove_override(struct page *page, struct gnttab_map_grant_ref *kmap_op, unsigned long mfn) { unsigned long flags; unsigned long pfn; unsigned long uninitialized_var(address); unsigned level; pte_t *ptep = NULL; pfn = page_to_pfn(page); if (!PageHighMem(page)) { address = (unsigned long)__va(pfn << PAGE_SHIFT); ptep = lookup_address(address, &level); if (WARN(ptep == NULL || level != PG_LEVEL_4K, "m2p_remove_override: pfn %lx not mapped", pfn)) return -EINVAL; } spin_lock_irqsave(&m2p_override_lock, flags); list_del(&page->lru); spin_unlock_irqrestore(&m2p_override_lock, flags); if (kmap_op != NULL) { if (!PageHighMem(page)) { struct multicall_space mcs; struct gnttab_unmap_and_replace *unmap_op; struct page *scratch_page = get_balloon_scratch_page(); unsigned long scratch_page_address = (unsigned long) __va(page_to_pfn(scratch_page) << PAGE_SHIFT); /* * It might be that we queued all the m2p grant table * hypercalls in a multicall, then m2p_remove_override * get called before the multicall has actually been * issued. In this case handle is going to -1 because * it hasn't been modified yet. */ if (kmap_op->handle == -1) xen_mc_flush(); /* * Now if kmap_op->handle is negative it means that the * hypercall actually returned an error. */ if (kmap_op->handle == GNTST_general_error) { pr_warn("m2p_remove_override: pfn %lx mfn %lx, failed to modify kernel mappings", pfn, mfn); put_balloon_scratch_page(); return -1; } xen_mc_batch(); mcs = __xen_mc_entry( sizeof(struct gnttab_unmap_and_replace)); unmap_op = mcs.args; unmap_op->host_addr = kmap_op->host_addr; unmap_op->new_addr = scratch_page_address; unmap_op->handle = kmap_op->handle; MULTI_grant_table_op(mcs.mc, GNTTABOP_unmap_and_replace, unmap_op, 1); mcs = __xen_mc_entry(0); MULTI_update_va_mapping(mcs.mc, scratch_page_address, pfn_pte(page_to_pfn(scratch_page), PAGE_KERNEL_RO), 0); xen_mc_issue(PARAVIRT_LAZY_MMU); kmap_op->host_addr = 0; put_balloon_scratch_page(); } } /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present * somewhere in this domain, even before being added to the * m2p_override (see comment above in m2p_add_override). * If there are no other entries in the m2p_override corresponding * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for * the original pfn (the one shared by the frontend): the backend * cannot do any IO on this page anymore because it has been * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of * the original pfn causes mfn_to_pfn(mfn) to return the frontend * pfn again. */ mfn &= ~FOREIGN_FRAME_BIT; pfn = mfn_to_pfn_no_overrides(mfn); if (__pfn_to_mfn(pfn) == FOREIGN_FRAME(mfn) && m2p_find_override(mfn) == NULL) set_phys_to_machine(pfn, mfn); return 0; }