int guest_remove_page(struct domain *d, unsigned long gmfn)
{
struct page_info *page;
#ifdef CONFIG_X86
p2m_type_t p2mt;
#endif
unsigned long mfn;
#ifdef CONFIG_X86
mfn = mfn_x(gfn_to_mfn(d, gmfn, &p2mt));
#else
mfn = gmfn_to_mfn(d, gmfn);
#endif
if ( unlikely(!mfn_valid(mfn)) )
{
gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
d->domain_id, gmfn);
return 0;
}
page = mfn_to_page(mfn);
#ifdef CONFIG_X86
/* If gmfn is shared, just drop the guest reference (which may or may not
* free the page) */
if(p2m_is_shared(p2mt))
{
put_page_and_type(page);
guest_physmap_remove_page(d, gmfn, mfn, 0);
return 1;
}
#endif /* CONFIG_X86 */
if ( unlikely(!get_page(page, d)) )
{
gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
return 0;
}
if ( test_and_clear_bit(_PGT_pinned, &page->u.inuse.type_info) )
put_page_and_type(page);
if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
put_page(page);
guest_physmap_remove_page(d, gmfn, mfn, 0);
put_page(page);
return 1;
}
static void
unshare_xenoprof_page_with_guest(struct xenoprof *x)
{
int i, npages = x->npages;
unsigned long mfn = virt_to_mfn(x->rawbuf);
for ( i = 0; i < npages; i++ )
{
struct page_info *page = mfn_to_page(mfn + i);
BUG_ON(page_get_owner(page) != current->domain);
if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
put_page(page);
}
}
static int
share_xenoprof_page_with_guest(struct domain *d, unsigned long mfn, int npages)
{
int i;
/* Check if previous page owner has released the page. */
for ( i = 0; i < npages; i++ )
{
struct page_info *page = mfn_to_page(mfn + i);
if ( (page->count_info & (PGC_allocated|PGC_count_mask)) != 0 )
{
gdprintk(XENLOG_INFO, "mfn 0x%lx page->count_info 0x%lx\n",
mfn + i, (unsigned long)page->count_info);
return -EBUSY;
}
page_set_owner(page, NULL);
}
for ( i = 0; i < npages; i++ )
share_xen_page_with_guest(mfn_to_page(mfn + i), d, XENSHARE_writable);
return 0;
}
static void teardown_apic_assist(struct vcpu *v)
{
void *va = v->arch.hvm_vcpu.viridian.apic_assist.va;
struct page_info *page;
if ( !va )
return;
v->arch.hvm_vcpu.viridian.apic_assist.va = NULL;
page = mfn_to_page(domain_page_map_to_mfn(va));
unmap_domain_page_global(va);
put_page_and_type(page);
}
/* Put any references on the single 4K page referenced by pte. TODO:
* Handle superpages, for now we only take special references for leaf
* pages (specifically foreign ones, which can't be super mapped today).
*/
static void p2m_put_l3_page(const lpae_t pte)
{
ASSERT(p2m_valid(pte));
/* TODO: Handle other p2m types
*
* It's safe to do the put_page here because page_alloc will
* flush the TLBs if the page is reallocated before the end of
* this loop.
*/
if ( p2m_is_foreign(pte.p2m.type) )
{
unsigned long mfn = pte.p2m.base;
ASSERT(mfn_valid(mfn));
put_page(mfn_to_page(mfn));
}
}
/* Free intermediate tables from a p2m sub-tree */
static void
p2m_free_entry(struct p2m_domain *p2m, l1_pgentry_t *p2m_entry, int page_order)
{
/* End if the entry is a leaf entry. */
if ( page_order == PAGE_ORDER_4K
|| !(l1e_get_flags(*p2m_entry) & _PAGE_PRESENT)
|| (l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
return;
if ( page_order > PAGE_ORDER_2M )
{
l1_pgentry_t *l3_table = map_domain_page(_mfn(l1e_get_pfn(*p2m_entry)));
for ( int i = 0; i < L3_PAGETABLE_ENTRIES; i++ )
p2m_free_entry(p2m, l3_table + i, page_order - 9);
unmap_domain_page(l3_table);
}
p2m_free_ptp(p2m, mfn_to_page(_mfn(l1e_get_pfn(*p2m_entry))));
}
/**
* p2m_mem_paging_nominate - Mark a guest page as to-be-paged-out
* @d: guest domain
* @gfn: guest page to nominate
*
* Returns 0 for success or negative errno values if gfn is not pageable.
*
* p2m_mem_paging_nominate() is called by the pager and checks if a guest page
* can be paged out. If the following conditions are met the p2mt will be
* changed:
* - the gfn is backed by a mfn
* - the p2mt of the gfn is pageable
* - the mfn is not used for IO
* - the mfn has exactly one user and has no special meaning
*
* Once the p2mt is changed the page is readonly for the guest. On success the
* pager can write the page contents to disk and later evict the page.
*/
int p2m_mem_paging_nominate(struct domain *d, unsigned long gfn)
{
struct page_info *page;
struct p2m_domain *p2m = p2m_get_hostp2m(d);
p2m_type_t p2mt;
p2m_access_t a;
mfn_t mfn;
int ret = -EBUSY;
gfn_lock(p2m, gfn, 0);
mfn = p2m->get_entry(p2m, gfn, &p2mt, &a, 0, NULL);
/* Check if mfn is valid */
if ( !mfn_valid(mfn) )
goto out;
/* Check p2m type */
if ( !p2m_is_pageable(p2mt) )
goto out;
/* Check for io memory page */
if ( is_iomem_page(mfn_x(mfn)) )
goto out;
/* Check page count and type */
page = mfn_to_page(mfn);
if ( (page->count_info & (PGC_count_mask | PGC_allocated)) !=
(1 | PGC_allocated) )
goto out;
if ( (page->u.inuse.type_info & PGT_count_mask) != 0 )
goto out;
/* Fix p2m entry */
set_p2m_entry(p2m, gfn, mfn, PAGE_ORDER_4K, p2m_ram_paging_out, a);
ret = 0;
out:
gfn_unlock(p2m, gfn, 0);
return ret;
}
int guest_remove_page(struct domain *d, unsigned long gmfn)
{
struct page_info *page;
#ifdef CONFIG_X86
p2m_type_t p2mt;
#endif
unsigned long mfn;
#ifdef CONFIG_X86
mfn = mfn_x(get_gfn_query(d, gmfn, &p2mt));
if ( unlikely(p2m_is_paging(p2mt)) )
{
guest_physmap_remove_page(d, gmfn, mfn, 0);
put_gfn(d, gmfn);
/* If the page hasn't yet been paged out, there is an
* actual page that needs to be released. */
if ( p2mt == p2m_ram_paging_out )
{
ASSERT(mfn_valid(mfn));
page = mfn_to_page(mfn);
if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
put_page(page);
}
p2m_mem_paging_drop_page(d, gmfn, p2mt);
return 1;
}
if ( p2mt == p2m_mmio_direct )
{
clear_mmio_p2m_entry(d, gmfn, _mfn(mfn));
put_gfn(d, gmfn);
return 1;
}
#else
mfn = gmfn_to_mfn(d, gmfn);
#endif
if ( unlikely(!mfn_valid(mfn)) )
{
put_gfn(d, gmfn);
gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
d->domain_id, gmfn);
return 0;
}
#ifdef CONFIG_X86
if ( p2m_is_shared(p2mt) )
{
/* Unshare the page, bail out on error. We unshare because
* we might be the only one using this shared page, and we
* need to trigger proper cleanup. Once done, this is
* like any other page. */
if ( mem_sharing_unshare_page(d, gmfn, 0) )
{
put_gfn(d, gmfn);
(void)mem_sharing_notify_enomem(d, gmfn, 0);
return 0;
}
/* Maybe the mfn changed */
mfn = mfn_x(get_gfn_query_unlocked(d, gmfn, &p2mt));
ASSERT(!p2m_is_shared(p2mt));
}
#endif /* CONFIG_X86 */
page = mfn_to_page(mfn);
if ( unlikely(!get_page(page, d)) )
{
put_gfn(d, gmfn);
gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
return 0;
}
if ( test_and_clear_bit(_PGT_pinned, &page->u.inuse.type_info) )
put_page_and_type(page);
/*
* With the lack of an IOMMU on some platforms, domains with DMA-capable
* device must retrieve the same pfn when the hypercall populate_physmap
* is called.
*
* For this purpose (and to match populate_physmap() behavior), the page
* is kept allocated.
*/
if ( !is_domain_direct_mapped(d) &&
test_and_clear_bit(_PGC_allocated, &page->count_info) )
put_page(page);
guest_physmap_remove_page(d, gmfn, mfn, 0);
put_page(page);
put_gfn(d, gmfn);
return 1;
}
static void populate_physmap(struct memop_args *a)
{
struct page_info *page;
unsigned int i, j;
xen_pfn_t gpfn, mfn;
struct domain *d = a->domain;
if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
a->nr_extents-1) )
return;
if ( a->extent_order > (a->memflags & MEMF_populate_on_demand ? MAX_ORDER :
max_order(current->domain)) )
return;
for ( i = a->nr_done; i < a->nr_extents; i++ )
{
if ( i != a->nr_done && hypercall_preempt_check() )
{
a->preempted = 1;
goto out;
}
if ( unlikely(__copy_from_guest_offset(&gpfn, a->extent_list, i, 1)) )
goto out;
if ( a->memflags & MEMF_populate_on_demand )
{
if ( guest_physmap_mark_populate_on_demand(d, gpfn,
a->extent_order) < 0 )
goto out;
}
else
{
if ( is_domain_direct_mapped(d) )
{
mfn = gpfn;
for ( j = 0; j < (1U << a->extent_order); j++, mfn++ )
{
if ( !mfn_valid(mfn) )
{
gdprintk(XENLOG_INFO, "Invalid mfn %#"PRI_xen_pfn"\n",
mfn);
goto out;
}
page = mfn_to_page(mfn);
if ( !get_page(page, d) )
{
gdprintk(XENLOG_INFO,
"mfn %#"PRI_xen_pfn" doesn't belong to d%d\n",
mfn, d->domain_id);
goto out;
}
put_page(page);
}
mfn = gpfn;
page = mfn_to_page(mfn);
}
else
{
page = alloc_domheap_pages(d, a->extent_order, a->memflags);
if ( unlikely(!page) )
{
if ( !opt_tmem || a->extent_order )
gdprintk(XENLOG_INFO,
"Could not allocate order=%u extent: id=%d memflags=%#x (%u of %u)\n",
a->extent_order, d->domain_id, a->memflags,
i, a->nr_extents);
goto out;
}
mfn = page_to_mfn(page);
}
guest_physmap_add_page(d, gpfn, mfn, a->extent_order);
if ( !paging_mode_translate(d) )
{
for ( j = 0; j < (1U << a->extent_order); j++ )
set_gpfn_from_mfn(mfn + j, gpfn + j);
/* Inform the domain of the new page's machine address. */
if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
goto out;
}
}
}
out:
a->nr_done = i;
}
static lpae_t mfn_to_p2m_entry(unsigned long mfn, unsigned int mattr,
p2m_type_t t)
{
paddr_t pa = ((paddr_t) mfn) << PAGE_SHIFT;
/* xn and write bit will be defined in the switch */
lpae_t e = (lpae_t) {
.p2m.af = 1,
.p2m.sh = LPAE_SH_OUTER,
.p2m.read = 1,
.p2m.mattr = mattr,
.p2m.table = 1,
.p2m.valid = 1,
.p2m.type = t,
};
BUILD_BUG_ON(p2m_max_real_type > (1 << 4));
switch (t)
{
case p2m_ram_rw:
e.p2m.xn = 0;
e.p2m.write = 1;
break;
case p2m_ram_ro:
e.p2m.xn = 0;
e.p2m.write = 0;
break;
case p2m_map_foreign:
case p2m_grant_map_rw:
case p2m_mmio_direct:
e.p2m.xn = 1;
e.p2m.write = 1;
break;
case p2m_grant_map_ro:
case p2m_invalid:
e.p2m.xn = 1;
e.p2m.write = 0;
break;
case p2m_max_real_type:
BUG();
break;
}
ASSERT(!(pa & ~PAGE_MASK));
ASSERT(!(pa & ~PADDR_MASK));
e.bits |= pa;
return e;
}
/* Allocate a new page table page and hook it in via the given entry */
static int p2m_create_table(struct domain *d,
lpae_t *entry)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
lpae_t pte;
BUG_ON(entry->p2m.valid);
page = alloc_domheap_page(NULL, 0);
if ( page == NULL )
return -ENOMEM;
page_list_add(page, &p2m->pages);
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM, p2m_invalid);
write_pte(entry, pte);
return 0;
}
enum p2m_operation {
INSERT,
ALLOCATE,
REMOVE,
RELINQUISH,
CACHEFLUSH,
};
static int apply_p2m_changes(struct domain *d,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t maddr,
int mattr,
p2m_type_t t)
{
int rc;
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t *first = NULL, *second = NULL, *third = NULL;
paddr_t addr;
unsigned long cur_first_page = ~0,
cur_first_offset = ~0,
cur_second_offset = ~0;
unsigned long count = 0;
unsigned int flush = 0;
bool_t populate = (op == INSERT || op == ALLOCATE);
lpae_t pte;
spin_lock(&p2m->lock);
if ( d != current->domain )
p2m_load_VTTBR(d);
addr = start_gpaddr;
while ( addr < end_gpaddr )
{
if ( cur_first_page != p2m_first_level_index(addr) )
{
if ( first ) unmap_domain_page(first);
first = p2m_map_first(p2m, addr);
if ( !first )
{
rc = -EINVAL;
goto out;
}
cur_first_page = p2m_first_level_index(addr);
}
if ( !first[first_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + FIRST_SIZE) & FIRST_MASK;
continue;
}
rc = p2m_create_table(d, &first[first_table_offset(addr)]);
if ( rc < 0 )
{
printk("p2m_populate_ram: L1 failed\n");
goto out;
}
}
BUG_ON(!first[first_table_offset(addr)].p2m.valid);
if ( cur_first_offset != first_table_offset(addr) )
{
if (second) unmap_domain_page(second);
second = map_domain_page(first[first_table_offset(addr)].p2m.base);
cur_first_offset = first_table_offset(addr);
}
/* else: second already valid */
if ( !second[second_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + SECOND_SIZE) & SECOND_MASK;
continue;
}
rc = p2m_create_table(d, &second[second_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L2 failed\n");
goto out;
}
}
BUG_ON(!second[second_table_offset(addr)].p2m.valid);
if ( cur_second_offset != second_table_offset(addr) )
{
/* map third level */
if (third) unmap_domain_page(third);
third = map_domain_page(second[second_table_offset(addr)].p2m.base);
cur_second_offset = second_table_offset(addr);
}
pte = third[third_table_offset(addr)];
flush |= pte.p2m.valid;
/* TODO: Handle other p2m type
*
* It's safe to do the put_page here because page_alloc will
* flush the TLBs if the page is reallocated before the end of
* this loop.
*/
if ( pte.p2m.valid && p2m_is_foreign(pte.p2m.type) )
{
unsigned long mfn = pte.p2m.base;
ASSERT(mfn_valid(mfn));
put_page(mfn_to_page(mfn));
}
/* Allocate a new RAM page and attach */
switch (op) {
case ALLOCATE:
{
struct page_info *page;
ASSERT(!pte.p2m.valid);
rc = -ENOMEM;
page = alloc_domheap_page(d, 0);
if ( page == NULL ) {
printk("p2m_populate_ram: failed to allocate page\n");
goto out;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
}
break;
case INSERT:
{
pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT, mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
case RELINQUISH:
case REMOVE:
{
if ( !pte.p2m.valid )
{
count++;
break;
}
count += 0x10;
memset(&pte, 0x00, sizeof(pte));
write_pte(&third[third_table_offset(addr)], pte);
count++;
}
break;
case CACHEFLUSH:
{
if ( !pte.p2m.valid || !p2m_is_ram(pte.p2m.type) )
break;
flush_page_to_ram(pte.p2m.base);
}
break;
}
/* Preempt every 2MiB (mapped) or 32 MiB (unmapped) - arbitrary */
if ( op == RELINQUISH && count >= 0x2000 )
{
if ( hypercall_preempt_check() )
{
p2m->lowest_mapped_gfn = addr >> PAGE_SHIFT;
rc = -EAGAIN;
goto out;
}
count = 0;
}
/* Got the next page */
addr += PAGE_SIZE;
}
if ( flush )
{
/* At the beginning of the function, Xen is updating VTTBR
* with the domain where the mappings are created. In this
* case it's only necessary to flush TLBs on every CPUs with
* the current VMID (our domain).
*/
flush_tlb();
}
if ( op == ALLOCATE || op == INSERT )
{
unsigned long sgfn = paddr_to_pfn(start_gpaddr);
unsigned long egfn = paddr_to_pfn(end_gpaddr);
p2m->max_mapped_gfn = MAX(p2m->max_mapped_gfn, egfn);
p2m->lowest_mapped_gfn = MIN(p2m->lowest_mapped_gfn, sgfn);
}
rc = 0;
out:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
if ( d != current->domain )
p2m_load_VTTBR(current->domain);
spin_unlock(&p2m->lock);
return rc;
}
/*
* Returns 0 if TLB flush / invalidate required by caller.
* va will indicate the address to be invalidated.
*
* addr is _either_ a host virtual address, or the address of the pte to
* update, as indicated by the GNTMAP_contains_pte flag.
*/
static void
__gnttab_map_grant_ref(
struct gnttab_map_grant_ref *op)
{
struct domain *ld, *rd;
struct vcpu *led;
int handle;
unsigned long frame = 0;
int rc = GNTST_okay;
unsigned int cache_flags;
struct active_grant_entry *act;
struct grant_mapping *mt;
grant_entry_t *sha;
union grant_combo scombo, prev_scombo, new_scombo;
/*
* We bound the number of times we retry CMPXCHG on memory locations that
* we share with a guest OS. The reason is that the guest can modify that
* location at a higher rate than we can read-modify-CMPXCHG, so the guest
* could cause us to livelock. There are a few cases where it is valid for
* the guest to race our updates (e.g., to change the GTF_readonly flag),
* so we allow a few retries before failing.
*/
int retries = 0;
led = current;
ld = led->domain;
if ( unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
{
gdprintk(XENLOG_INFO, "Bad flags in grant map op (%x).\n", op->flags);
op->status = GNTST_bad_gntref;
return;
}
if ( unlikely((rd = rcu_lock_domain_by_id(op->dom)) == NULL) )
{
gdprintk(XENLOG_INFO, "Could not find domain %d\n", op->dom);
op->status = GNTST_bad_domain;
return;
}
rc = xsm_grant_mapref(ld, rd, op->flags);
if ( rc )
{
rcu_unlock_domain(rd);
op->status = GNTST_permission_denied;
return;
}
if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
{
rcu_unlock_domain(rd);
gdprintk(XENLOG_INFO, "Failed to obtain maptrack handle.\n");
op->status = GNTST_no_device_space;
return;
}
spin_lock(&rd->grant_table->lock);
/* Bounds check on the grant ref */
if ( unlikely(op->ref >= nr_grant_entries(rd->grant_table)))
PIN_FAIL(unlock_out, GNTST_bad_gntref, "Bad ref (%d).\n", op->ref);
act = &active_entry(rd->grant_table, op->ref);
sha = &shared_entry(rd->grant_table, op->ref);
/* If already pinned, check the active domid and avoid refcnt overflow. */
if ( act->pin &&
((act->domid != ld->domain_id) ||
(act->pin & 0x80808080U) != 0) )
PIN_FAIL(unlock_out, GNTST_general_error,
"Bad domain (%d != %d), or risk of counter overflow %08x\n",
act->domid, ld->domain_id, act->pin);
if ( !act->pin ||
(!(op->flags & GNTMAP_readonly) &&
!(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
{
scombo.word = *(u32 *)&sha->flags;
/*
* This loop attempts to set the access (reading/writing) flags
* in the grant table entry. It tries a cmpxchg on the field
* up to five times, and then fails under the assumption that
* the guest is misbehaving.
*/
for ( ; ; )
{
/* If not already pinned, check the grant domid and type. */
if ( !act->pin &&
(((scombo.shorts.flags & GTF_type_mask) !=
GTF_permit_access) ||
(scombo.shorts.domid != ld->domain_id)) )
PIN_FAIL(unlock_out, GNTST_general_error,
"Bad flags (%x) or dom (%d). (expected dom %d)\n",
scombo.shorts.flags, scombo.shorts.domid,
ld->domain_id);
new_scombo = scombo;
new_scombo.shorts.flags |= GTF_reading;
if ( !(op->flags & GNTMAP_readonly) )
{
new_scombo.shorts.flags |= GTF_writing;
if ( unlikely(scombo.shorts.flags & GTF_readonly) )
PIN_FAIL(unlock_out, GNTST_general_error,
"Attempt to write-pin a r/o grant entry.\n");
}
prev_scombo.word = cmpxchg((u32 *)&sha->flags,
scombo.word, new_scombo.word);
if ( likely(prev_scombo.word == scombo.word) )
break;
if ( retries++ == 4 )
PIN_FAIL(unlock_out, GNTST_general_error,
"Shared grant entry is unstable.\n");
scombo = prev_scombo;
}
if ( !act->pin )
{
act->domid = scombo.shorts.domid;
act->frame = gmfn_to_mfn(rd, sha->frame);
}
}
if ( op->flags & GNTMAP_device_map )
act->pin += (op->flags & GNTMAP_readonly) ?
GNTPIN_devr_inc : GNTPIN_devw_inc;
if ( op->flags & GNTMAP_host_map )
act->pin += (op->flags & GNTMAP_readonly) ?
GNTPIN_hstr_inc : GNTPIN_hstw_inc;
frame = act->frame;
cache_flags = (sha->flags & (GTF_PAT | GTF_PWT | GTF_PCD) );
spin_unlock(&rd->grant_table->lock);
if ( is_iomem_page(frame) )
{
if ( !iomem_access_permitted(rd, frame, frame) )
{
gdprintk(XENLOG_WARNING,
"Iomem mapping not permitted %lx (domain %d)\n",
frame, rd->domain_id);
rc = GNTST_general_error;
goto undo_out;
}
rc = create_grant_host_mapping(
op->host_addr, frame, op->flags, cache_flags);
if ( rc != GNTST_okay )
goto undo_out;
}
else
{
if ( unlikely(!mfn_valid(frame)) ||
unlikely(!(gnttab_host_mapping_get_page_type(op, ld, rd) ?
get_page_and_type(mfn_to_page(frame), rd,
PGT_writable_page) :
get_page(mfn_to_page(frame), rd))) )
{
if ( !rd->is_dying )
gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n",
frame);
rc = GNTST_general_error;
goto undo_out;
}
if ( op->flags & GNTMAP_host_map )
{
rc = create_grant_host_mapping(op->host_addr, frame, op->flags, 0);
if ( rc != GNTST_okay )
{
if ( gnttab_host_mapping_get_page_type(op, ld, rd) )
put_page_type(mfn_to_page(frame));
put_page(mfn_to_page(frame));
goto undo_out;
}
if ( op->flags & GNTMAP_device_map )
{
(void)get_page(mfn_to_page(frame), rd);
if ( !(op->flags & GNTMAP_readonly) )
get_page_type(mfn_to_page(frame), PGT_writable_page);
}
}
}
TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
mt = &maptrack_entry(ld->grant_table, handle);
mt->domid = op->dom;
mt->ref = op->ref;
mt->flags = op->flags;
op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
op->handle = handle;
op->status = GNTST_okay;
rcu_unlock_domain(rd);
return;
undo_out:
spin_lock(&rd->grant_table->lock);
act = &active_entry(rd->grant_table, op->ref);
sha = &shared_entry(rd->grant_table, op->ref);
if ( op->flags & GNTMAP_device_map )
act->pin -= (op->flags & GNTMAP_readonly) ?
GNTPIN_devr_inc : GNTPIN_devw_inc;
if ( op->flags & GNTMAP_host_map )
act->pin -= (op->flags & GNTMAP_readonly) ?
GNTPIN_hstr_inc : GNTPIN_hstw_inc;
if ( !(op->flags & GNTMAP_readonly) &&
!(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
gnttab_clear_flag(_GTF_writing, &sha->flags);
if ( !act->pin )
gnttab_clear_flag(_GTF_reading, &sha->flags);
unlock_out:
spin_unlock(&rd->grant_table->lock);
op->status = rc;
put_maptrack_handle(ld->grant_table, handle);
rcu_unlock_domain(rd);
}
/*
* If mem_access is in use it might have been the reason why get_page_from_gva
* failed to fetch the page, as it uses the MMU for the permission checking.
* Only in these cases we do a software-based type check and fetch the page if
* we indeed found a conflicting mem_access setting.
*/
struct page_info*
p2m_mem_access_check_and_get_page(vaddr_t gva, unsigned long flag,
const struct vcpu *v)
{
long rc;
paddr_t ipa;
gfn_t gfn;
mfn_t mfn;
xenmem_access_t xma;
p2m_type_t t;
struct page_info *page = NULL;
struct p2m_domain *p2m = &v->domain->arch.p2m;
rc = gva_to_ipa(gva, &ipa, flag);
if ( rc < 0 )
goto err;
gfn = _gfn(paddr_to_pfn(ipa));
/*
* We do this first as this is faster in the default case when no
* permission is set on the page.
*/
rc = __p2m_get_mem_access(v->domain, gfn, &xma);
if ( rc < 0 )
goto err;
/* Let's check if mem_access limited the access. */
switch ( xma )
{
default:
case XENMEM_access_rwx:
case XENMEM_access_rw:
/*
* If mem_access contains no rw perm restrictions at all then the original
* fault was correct.
*/
goto err;
case XENMEM_access_n2rwx:
case XENMEM_access_n:
case XENMEM_access_x:
/*
* If no r/w is permitted by mem_access, this was a fault caused by mem_access.
*/
break;
case XENMEM_access_wx:
case XENMEM_access_w:
/*
* If this was a read then it was because of mem_access, but if it was
* a write then the original get_page_from_gva fault was correct.
*/
if ( flag == GV2M_READ )
break;
else
goto err;
case XENMEM_access_rx2rw:
case XENMEM_access_rx:
case XENMEM_access_r:
/*
* If this was a write then it was because of mem_access, but if it was
* a read then the original get_page_from_gva fault was correct.
*/
if ( flag == GV2M_WRITE )
break;
else
goto err;
}
/*
* We had a mem_access permission limiting the access, but the page type
* could also be limiting, so we need to check that as well.
*/
mfn = p2m_get_entry(p2m, gfn, &t, NULL, NULL);
if ( mfn_eq(mfn, INVALID_MFN) )
goto err;
if ( !mfn_valid(mfn) )
goto err;
/*
* Base type doesn't allow r/w
*/
if ( t != p2m_ram_rw )
goto err;
page = mfn_to_page(mfn_x(mfn));
if ( unlikely(!get_page(page, v->domain)) )
page = NULL;
err:
return page;
}
int guest_remove_page(struct domain *d, unsigned long gmfn)
{
struct page_info *page;
#ifdef CONFIG_X86
p2m_type_t p2mt;
#endif
unsigned long mfn;
#ifdef CONFIG_X86
mfn = mfn_x(get_gfn_query(d, gmfn, &p2mt));
if ( unlikely(p2m_is_paging(p2mt)) )
{
guest_physmap_remove_page(d, gmfn, mfn, 0);
put_gfn(d, gmfn);
/* If the page hasn't yet been paged out, there is an
* actual page that needs to be released. */
if ( p2mt == p2m_ram_paging_out )
{
ASSERT(mfn_valid(mfn));
page = mfn_to_page(mfn);
if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
put_page(page);
}
p2m_mem_paging_drop_page(d, gmfn, p2mt);
return 1;
}
#else
mfn = gmfn_to_mfn(d, gmfn);
#endif
if ( unlikely(!mfn_valid(mfn)) )
{
put_gfn(d, gmfn);
gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
d->domain_id, gmfn);
return 0;
}
#ifdef CONFIG_X86_64
if ( p2m_is_shared(p2mt) )
{
/* Unshare the page, bail out on error. We unshare because
* we might be the only one using this shared page, and we
* need to trigger proper cleanup. Once done, this is
* like any other page. */
if ( mem_sharing_unshare_page(d, gmfn, 0) )
{
put_gfn(d, gmfn);
(void)mem_sharing_notify_enomem(d, gmfn, 0);
return 0;
}
/* Maybe the mfn changed */
mfn = mfn_x(get_gfn_query_unlocked(d, gmfn, &p2mt));
ASSERT(!p2m_is_shared(p2mt));
}
#endif /* CONFIG_X86_64 */
page = mfn_to_page(mfn);
if ( unlikely(!get_page(page, d)) )
{
put_gfn(d, gmfn);
gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
return 0;
}
if ( test_and_clear_bit(_PGT_pinned, &page->u.inuse.type_info) )
put_page_and_type(page);
if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
put_page(page);
guest_physmap_remove_page(d, gmfn, mfn, 0);
put_page(page);
put_gfn(d, gmfn);
return 1;
}
int
guest_physmap_add_entry(struct domain *d, unsigned long gfn,
unsigned long mfn, unsigned int page_order,
p2m_type_t t)
{
struct p2m_domain *p2m = p2m_get_hostp2m(d);
unsigned long i, ogfn;
p2m_type_t ot;
p2m_access_t a;
mfn_t omfn;
int pod_count = 0;
int rc = 0;
if ( !paging_mode_translate(d) )
{
if ( need_iommu(d) && t == p2m_ram_rw )
{
for ( i = 0; i < (1 << page_order); i++ )
{
rc = iommu_map_page(
d, mfn + i, mfn + i, IOMMUF_readable|IOMMUF_writable);
if ( rc != 0 )
{
while ( i-- > 0 )
iommu_unmap_page(d, mfn + i);
return rc;
}
}
}
return 0;
}
p2m_lock(p2m);
P2M_DEBUG("adding gfn=%#lx mfn=%#lx\n", gfn, mfn);
/* First, remove m->p mappings for existing p->m mappings */
for ( i = 0; i < (1UL << page_order); i++ )
{
omfn = p2m->get_entry(p2m, gfn + i, &ot, &a, 0, NULL);
if ( p2m_is_shared(ot) )
{
/* Do an unshare to cleanly take care of all corner
* cases. */
int rc;
rc = mem_sharing_unshare_page(p2m->domain, gfn + i, 0);
if ( rc )
{
p2m_unlock(p2m);
/* NOTE: Should a guest domain bring this upon itself,
* there is not a whole lot we can do. We are buried
* deep in locks from most code paths by now. So, fail
* the call and don't try to sleep on a wait queue
* while placing the mem event.
*
* However, all current (changeset 3432abcf9380) code
* paths avoid this unsavoury situation. For now.
*
* Foreign domains are okay to place an event as they
* won't go to sleep. */
(void)mem_sharing_notify_enomem(p2m->domain, gfn + i, 0);
return rc;
}
omfn = p2m->get_entry(p2m, gfn + i, &ot, &a, 0, NULL);
ASSERT(!p2m_is_shared(ot));
}
if ( p2m_is_grant(ot) )
{
/* Really shouldn't be unmapping grant maps this way */
domain_crash(d);
p2m_unlock(p2m);
return -EINVAL;
}
else if ( p2m_is_ram(ot) && !p2m_is_paged(ot) )
{
ASSERT(mfn_valid(omfn));
set_gpfn_from_mfn(mfn_x(omfn), INVALID_M2P_ENTRY);
}
else if ( ot == p2m_populate_on_demand )
{
/* Count how man PoD entries we'll be replacing if successful */
pod_count++;
}
else if ( p2m_is_paging(ot) && (ot != p2m_ram_paging_out) )
{
/* We're plugging a hole in the physmap where a paged out page was */
atomic_dec(&d->paged_pages);
}
}
/* Then, look for m->p mappings for this range and deal with them */
for ( i = 0; i < (1UL << page_order); i++ )
{
if ( page_get_owner(mfn_to_page(_mfn(mfn + i))) == dom_cow )
{
/* This is no way to add a shared page to your physmap! */
gdprintk(XENLOG_ERR, "Adding shared mfn %lx directly to dom %hu "
"physmap not allowed.\n", mfn+i, d->domain_id);
p2m_unlock(p2m);
return -EINVAL;
}
if ( page_get_owner(mfn_to_page(_mfn(mfn + i))) != d )
continue;
ogfn = mfn_to_gfn(d, _mfn(mfn+i));
if ( (ogfn != INVALID_M2P_ENTRY) && (ogfn != gfn + i) )
{
/* This machine frame is already mapped at another physical
* address */
P2M_DEBUG("aliased! mfn=%#lx, old gfn=%#lx, new gfn=%#lx\n",
mfn + i, ogfn, gfn + i);
omfn = p2m->get_entry(p2m, ogfn, &ot, &a, 0, NULL);
if ( p2m_is_ram(ot) && !p2m_is_paged(ot) )
{
ASSERT(mfn_valid(omfn));
P2M_DEBUG("old gfn=%#lx -> mfn %#lx\n",
ogfn , mfn_x(omfn));
if ( mfn_x(omfn) == (mfn + i) )
p2m_remove_page(p2m, ogfn, mfn + i, 0);
}
}
}
/* Now, actually do the two-way mapping */
if ( mfn_valid(_mfn(mfn)) )
{
if ( !set_p2m_entry(p2m, gfn, _mfn(mfn), page_order, t, p2m->default_access) )
{
rc = -EINVAL;
goto out; /* Failed to update p2m, bail without updating m2p. */
}
if ( !p2m_is_grant(t) )
{
for ( i = 0; i < (1UL << page_order); i++ )
set_gpfn_from_mfn(mfn+i, gfn+i);
}
}
else
{
gdprintk(XENLOG_WARNING, "Adding bad mfn to p2m map (%#lx -> %#lx)\n",
gfn, mfn);
if ( !set_p2m_entry(p2m, gfn, _mfn(INVALID_MFN), page_order,
p2m_invalid, p2m->default_access) )
rc = -EINVAL;
else
{
pod_lock(p2m);
p2m->pod.entry_count -= pod_count;
BUG_ON(p2m->pod.entry_count < 0);
pod_unlock(p2m);
}
}
out:
p2m_unlock(p2m);
return rc;
}
