/* Atomically look up a GFN and take a reference count on the backing page. */
struct page_info *get_page_from_gfn_p2m(
struct domain *d, struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q)
{
struct page_info *page = NULL;
p2m_access_t _a;
p2m_type_t _t;
mfn_t mfn;
/* Allow t or a to be NULL */
t = t ?: &_t;
a = a ?: &_a;
if ( likely(!p2m_locked_by_me(p2m)) )
{
/* Fast path: look up and get out */
p2m_read_lock(p2m);
mfn = __get_gfn_type_access(p2m, gfn, t, a, 0, NULL, 0);
if ( (p2m_is_ram(*t) || p2m_is_grant(*t))
&& mfn_valid(mfn)
&& !((q & P2M_UNSHARE) && p2m_is_shared(*t)) )
{
page = mfn_to_page(mfn);
if ( !get_page(page, d)
/* Page could be shared */
&& !get_page(page, dom_cow) )
page = NULL;
}
p2m_read_unlock(p2m);
if ( page )
return page;
/* Error path: not a suitable GFN at all */
if ( !p2m_is_ram(*t) && !p2m_is_paging(*t) && !p2m_is_pod(*t) )
return NULL;
}
/* Slow path: take the write lock and do fixups */
mfn = get_gfn_type_access(p2m, gfn, t, a, q, NULL);
if ( p2m_is_ram(*t) && mfn_valid(mfn) )
{
page = mfn_to_page(mfn);
if ( !get_page(page, d) )
page = NULL;
}
put_gfn(d, gfn);
return page;
}
int
set_mmio_p2m_entry(struct domain *d, unsigned long gfn, mfn_t mfn)
{
int rc = 0;
p2m_access_t a;
p2m_type_t ot;
mfn_t omfn;
struct p2m_domain *p2m = p2m_get_hostp2m(d);
if ( !paging_mode_translate(d) )
return 0;
gfn_lock(p2m, gfn, 0);
omfn = p2m->get_entry(p2m, gfn, &ot, &a, 0, NULL);
if ( p2m_is_grant(ot) )
{
p2m_unlock(p2m);
domain_crash(d);
return 0;
}
else if ( p2m_is_ram(ot) )
{
ASSERT(mfn_valid(omfn));
set_gpfn_from_mfn(mfn_x(omfn), INVALID_M2P_ENTRY);
}
P2M_DEBUG("set mmio %lx %lx\n", gfn, mfn_x(mfn));
rc = set_p2m_entry(p2m, gfn, mfn, PAGE_ORDER_4K, p2m_mmio_direct, p2m->default_access);
gfn_unlock(p2m, gfn, 0);
if ( 0 == rc )
gdprintk(XENLOG_ERR,
"set_mmio_p2m_entry: set_p2m_entry failed! mfn=%08lx\n",
mfn_x(get_gfn_query_unlocked(p2m->domain, gfn, &ot)));
return rc;
}
unsigned long hap_gva_to_gfn(GUEST_PAGING_LEVELS)(
struct vcpu *v, unsigned long gva, uint32_t *pfec)
{
unsigned long cr3;
uint32_t missing;
mfn_t top_mfn;
void *top_map;
p2m_type_t p2mt;
walk_t gw;
/* Get the top-level table's MFN */
cr3 = v->arch.hvm_vcpu.guest_cr[3];
top_mfn = gfn_to_mfn(v->domain, _gfn(cr3 >> PAGE_SHIFT), &p2mt);
if ( !p2m_is_ram(p2mt) )
{
pfec[0] &= ~PFEC_page_present;
return INVALID_GFN;
}
/* Map the top-level table and call the tree-walker */
ASSERT(mfn_valid(mfn_x(top_mfn)));
top_map = map_domain_page(mfn_x(top_mfn));
#if GUEST_PAGING_LEVELS == 3
top_map += (cr3 & ~(PAGE_MASK | 31));
#endif
missing = guest_walk_tables(v, gva, &gw, pfec[0], top_mfn, top_map);
unmap_domain_page(top_map);
/* Interpret the answer */
if ( missing == 0 )
return gfn_x(guest_l1e_get_gfn(gw.l1e));
if ( missing & _PAGE_PRESENT )
pfec[0] &= ~PFEC_page_present;
return INVALID_GFN;
}
static lpae_t mfn_to_p2m_entry(unsigned long mfn, unsigned int mattr,
p2m_type_t t)
{
paddr_t pa = ((paddr_t) mfn) << PAGE_SHIFT;
/* xn and write bit will be defined in the switch */
lpae_t e = (lpae_t) {
.p2m.af = 1,
.p2m.sh = LPAE_SH_OUTER,
.p2m.read = 1,
.p2m.mattr = mattr,
.p2m.table = 1,
.p2m.valid = 1,
.p2m.type = t,
};
BUILD_BUG_ON(p2m_max_real_type > (1 << 4));
switch (t)
{
case p2m_ram_rw:
e.p2m.xn = 0;
e.p2m.write = 1;
break;
case p2m_ram_ro:
e.p2m.xn = 0;
e.p2m.write = 0;
break;
case p2m_map_foreign:
case p2m_grant_map_rw:
case p2m_mmio_direct:
e.p2m.xn = 1;
e.p2m.write = 1;
break;
case p2m_grant_map_ro:
case p2m_invalid:
e.p2m.xn = 1;
e.p2m.write = 0;
break;
case p2m_max_real_type:
BUG();
break;
}
ASSERT(!(pa & ~PAGE_MASK));
ASSERT(!(pa & ~PADDR_MASK));
e.bits |= pa;
return e;
}
/* Allocate a new page table page and hook it in via the given entry */
static int p2m_create_table(struct domain *d,
lpae_t *entry)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
lpae_t pte;
BUG_ON(entry->p2m.valid);
page = alloc_domheap_page(NULL, 0);
if ( page == NULL )
return -ENOMEM;
page_list_add(page, &p2m->pages);
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM, p2m_invalid);
write_pte(entry, pte);
return 0;
}
enum p2m_operation {
INSERT,
ALLOCATE,
REMOVE,
RELINQUISH,
CACHEFLUSH,
};
static int apply_p2m_changes(struct domain *d,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t maddr,
int mattr,
p2m_type_t t)
{
int rc;
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t *first = NULL, *second = NULL, *third = NULL;
paddr_t addr;
unsigned long cur_first_page = ~0,
cur_first_offset = ~0,
cur_second_offset = ~0;
unsigned long count = 0;
unsigned int flush = 0;
bool_t populate = (op == INSERT || op == ALLOCATE);
lpae_t pte;
spin_lock(&p2m->lock);
if ( d != current->domain )
p2m_load_VTTBR(d);
addr = start_gpaddr;
while ( addr < end_gpaddr )
{
if ( cur_first_page != p2m_first_level_index(addr) )
{
if ( first ) unmap_domain_page(first);
first = p2m_map_first(p2m, addr);
if ( !first )
{
rc = -EINVAL;
goto out;
}
cur_first_page = p2m_first_level_index(addr);
}
if ( !first[first_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + FIRST_SIZE) & FIRST_MASK;
continue;
}
rc = p2m_create_table(d, &first[first_table_offset(addr)]);
if ( rc < 0 )
{
printk("p2m_populate_ram: L1 failed\n");
goto out;
}
}
BUG_ON(!first[first_table_offset(addr)].p2m.valid);
if ( cur_first_offset != first_table_offset(addr) )
{
if (second) unmap_domain_page(second);
second = map_domain_page(first[first_table_offset(addr)].p2m.base);
cur_first_offset = first_table_offset(addr);
}
/* else: second already valid */
if ( !second[second_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + SECOND_SIZE) & SECOND_MASK;
continue;
}
rc = p2m_create_table(d, &second[second_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L2 failed\n");
goto out;
}
}
BUG_ON(!second[second_table_offset(addr)].p2m.valid);
if ( cur_second_offset != second_table_offset(addr) )
{
/* map third level */
if (third) unmap_domain_page(third);
third = map_domain_page(second[second_table_offset(addr)].p2m.base);
cur_second_offset = second_table_offset(addr);
}
pte = third[third_table_offset(addr)];
flush |= pte.p2m.valid;
/* TODO: Handle other p2m type
*
* It's safe to do the put_page here because page_alloc will
* flush the TLBs if the page is reallocated before the end of
* this loop.
*/
if ( pte.p2m.valid && p2m_is_foreign(pte.p2m.type) )
{
unsigned long mfn = pte.p2m.base;
ASSERT(mfn_valid(mfn));
put_page(mfn_to_page(mfn));
}
/* Allocate a new RAM page and attach */
switch (op) {
case ALLOCATE:
{
struct page_info *page;
ASSERT(!pte.p2m.valid);
rc = -ENOMEM;
page = alloc_domheap_page(d, 0);
if ( page == NULL ) {
printk("p2m_populate_ram: failed to allocate page\n");
goto out;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
}
break;
case INSERT:
{
pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT, mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
case RELINQUISH:
case REMOVE:
{
if ( !pte.p2m.valid )
{
count++;
break;
}
count += 0x10;
memset(&pte, 0x00, sizeof(pte));
write_pte(&third[third_table_offset(addr)], pte);
count++;
}
break;
case CACHEFLUSH:
{
if ( !pte.p2m.valid || !p2m_is_ram(pte.p2m.type) )
break;
flush_page_to_ram(pte.p2m.base);
}
break;
}
/* Preempt every 2MiB (mapped) or 32 MiB (unmapped) - arbitrary */
if ( op == RELINQUISH && count >= 0x2000 )
{
if ( hypercall_preempt_check() )
{
p2m->lowest_mapped_gfn = addr >> PAGE_SHIFT;
rc = -EAGAIN;
goto out;
}
count = 0;
}
/* Got the next page */
addr += PAGE_SIZE;
}
if ( flush )
{
/* At the beginning of the function, Xen is updating VTTBR
* with the domain where the mappings are created. In this
* case it's only necessary to flush TLBs on every CPUs with
* the current VMID (our domain).
*/
flush_tlb();
}
if ( op == ALLOCATE || op == INSERT )
{
unsigned long sgfn = paddr_to_pfn(start_gpaddr);
unsigned long egfn = paddr_to_pfn(end_gpaddr);
p2m->max_mapped_gfn = MAX(p2m->max_mapped_gfn, egfn);
p2m->lowest_mapped_gfn = MIN(p2m->lowest_mapped_gfn, sgfn);
}
rc = 0;
out:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
if ( d != current->domain )
p2m_load_VTTBR(current->domain);
spin_unlock(&p2m->lock);
return rc;
}
static mfn_t
p2m_gfn_to_mfn(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
unsigned long l1e_flags;
p2m_type_t l1t;
ASSERT(paging_mode_translate(p2m->domain));
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
*t = p2m_mmio_dm;
/* Not implemented except with EPT */
*a = p2m_access_rwx;
if ( gfn > p2m->max_mapped_pfn )
/* This pfn is higher than the highest the p2m map currently holds */
return _mfn(INVALID_MFN);
/* Use the fast path with the linear mapping if we can */
if ( p2m == p2m_get_hostp2m(current->domain) )
return p2m_gfn_to_mfn_current(p2m, gfn, t, a, q, page_order);
mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
#if CONFIG_PAGING_LEVELS >= 4
{
l4_pgentry_t *l4e = map_domain_page(mfn_x(mfn));
l4e += l4_table_offset(addr);
if ( (l4e_get_flags(*l4e) & _PAGE_PRESENT) == 0 )
{
unmap_domain_page(l4e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l4e_get_pfn(*l4e));
unmap_domain_page(l4e);
}
#endif
{
l3_pgentry_t *l3e = map_domain_page(mfn_x(mfn));
#if CONFIG_PAGING_LEVELS == 3
/* On PAE hosts the p2m has eight l3 entries, not four (see
* shadow_set_p2m_entry()) so we can't use l3_table_offset.
* Instead, just count the number of l3es from zero. It's safe
* to do this because we already checked that the gfn is within
* the bounds of the p2m. */
l3e += (addr >> L3_PAGETABLE_SHIFT);
#else
l3e += l3_table_offset(addr);
#endif
pod_retry_l3:
if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
{
if ( p2m_flags_to_type(l3e_get_flags(*l3e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
}
else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l3e);
return _mfn(INVALID_MFN);
}
else if ( (l3e_get_flags(*l3e) & _PAGE_PSE) )
{
mfn = _mfn(l3e_get_pfn(*l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
*t = p2m_flags_to_type(l3e_get_flags(*l3e));
unmap_domain_page(l3e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_1G;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l3e_get_pfn(*l3e));
unmap_domain_page(l3e);
}
l2e = map_domain_page(mfn_x(mfn));
l2e += l2_table_offset(addr);
pod_retry_l2:
if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )
{
/* PoD: Try to populate a 2-meg chunk */
if ( p2m_flags_to_type(l2e_get_flags(*l2e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_2M, q) )
goto pod_retry_l2;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l2e);
return _mfn(INVALID_MFN);
}
else if ( (l2e_get_flags(*l2e) & _PAGE_PSE) )
{
mfn = _mfn(l2e_get_pfn(*l2e) + l1_table_offset(addr));
*t = p2m_flags_to_type(l2e_get_flags(*l2e));
unmap_domain_page(l2e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_2M;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l2e_get_pfn(*l2e));
unmap_domain_page(l2e);
l1e = map_domain_page(mfn_x(mfn));
l1e += l1_table_offset(addr);
pod_retry_l1:
l1e_flags = l1e_get_flags(*l1e);
l1t = p2m_flags_to_type(l1e_flags);
if ( ((l1e_flags & _PAGE_PRESENT) == 0) && (!p2m_is_paging(l1t)) )
{
/* PoD: Try to populate */
if ( l1t == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_4K, q) )
goto pod_retry_l1;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l1e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l1e_get_pfn(*l1e));
*t = l1t;
unmap_domain_page(l1e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t) || p2m_is_paging(*t));
if ( page_order )
*page_order = PAGE_ORDER_4K;
return (p2m_is_valid(*t) || p2m_is_grant(*t)) ? mfn : _mfn(INVALID_MFN);
}
/* Read the current domain's p2m table (through the linear mapping). */
static mfn_t p2m_gfn_to_mfn_current(struct p2m_domain *p2m,
unsigned long gfn, p2m_type_t *t,
p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn = _mfn(INVALID_MFN);
p2m_type_t p2mt = p2m_mmio_dm;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
l1_pgentry_t l1e = l1e_empty(), *p2m_entry;
l2_pgentry_t l2e = l2e_empty();
int ret;
#if CONFIG_PAGING_LEVELS >= 4
l3_pgentry_t l3e = l3e_empty();
#endif
ASSERT(gfn < (RO_MPT_VIRT_END - RO_MPT_VIRT_START)
/ sizeof(l1_pgentry_t));
#if CONFIG_PAGING_LEVELS >= 4
/*
* Read & process L3
*/
p2m_entry = (l1_pgentry_t *)
&__linear_l2_table[l2_linear_offset(RO_MPT_VIRT_START)
+ l3_linear_offset(addr)];
pod_retry_l3:
ret = __copy_from_user(&l3e, p2m_entry, sizeof(l3e));
if ( ret != 0 || !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
{
if ( (l3e_get_flags(l3e) & _PAGE_PSE) &&
(p2m_flags_to_type(l3e_get_flags(l3e)) == p2m_populate_on_demand) )
{
/* The read has succeeded, so we know that mapping exists */
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
p2mt = p2m_invalid;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
goto out;
}
else
{
p2mt = p2m_populate_on_demand;
goto out;
}
}
goto pod_retry_l2;
}
if ( l3e_get_flags(l3e) & _PAGE_PSE )
{
p2mt = p2m_flags_to_type(l3e_get_flags(l3e));
ASSERT(l3e_get_pfn(l3e) != INVALID_MFN || !p2m_is_ram(p2mt));
if (p2m_is_valid(p2mt) )
mfn = _mfn(l3e_get_pfn(l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
else
p2mt = p2m_mmio_dm;
if ( page_order )
*page_order = PAGE_ORDER_1G;
goto out;
}
#endif
/*
* Read & process L2
*/
p2m_entry = &__linear_l1_table[l1_linear_offset(RO_MPT_VIRT_START)
+ l2_linear_offset(addr)];
pod_retry_l2:
ret = __copy_from_user(&l2e,
p2m_entry,
sizeof(l2e));
if ( ret != 0
|| !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
{
if( (l2e_get_flags(l2e) & _PAGE_PSE)
&& ( p2m_flags_to_type(l2e_get_flags(l2e))
== p2m_populate_on_demand ) )
{
/* The read has succeeded, so we know that the mapping
* exits at this point. */
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn,
PAGE_ORDER_2M, q) )
goto pod_retry_l2;
/* Allocate failed. */
p2mt = p2m_invalid;
printk("%s: Allocate failed!\n", __func__);
goto out;
}
else
{
p2mt = p2m_populate_on_demand;
goto out;
}
}
goto pod_retry_l1;
}
if (l2e_get_flags(l2e) & _PAGE_PSE)
{
p2mt = p2m_flags_to_type(l2e_get_flags(l2e));
ASSERT(l2e_get_pfn(l2e) != INVALID_MFN || !p2m_is_ram(p2mt));
if ( p2m_is_valid(p2mt) )
mfn = _mfn(l2e_get_pfn(l2e) + l1_table_offset(addr));
else
p2mt = p2m_mmio_dm;
if ( page_order )
*page_order = PAGE_ORDER_2M;
goto out;
}
/*
* Read and process L1
*/
/* Need to __copy_from_user because the p2m is sparse and this
* part might not exist */
pod_retry_l1:
p2m_entry = &phys_to_machine_mapping[gfn];
ret = __copy_from_user(&l1e,
p2m_entry,
sizeof(l1e));
if ( ret == 0 ) {
unsigned long l1e_mfn = l1e_get_pfn(l1e);
p2mt = p2m_flags_to_type(l1e_get_flags(l1e));
ASSERT( mfn_valid(_mfn(l1e_mfn)) || !p2m_is_ram(p2mt) ||
p2m_is_paging(p2mt) );
if ( p2mt == p2m_populate_on_demand )
{
/* The read has succeeded, so we know that the mapping
* exits at this point. */
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn,
PAGE_ORDER_4K, q) )
goto pod_retry_l1;
/* Allocate failed. */
p2mt = p2m_invalid;
goto out;
}
else
{
p2mt = p2m_populate_on_demand;
goto out;
}
}
if ( p2m_is_valid(p2mt) || p2m_is_grant(p2mt) )
mfn = _mfn(l1e_mfn);
else
/* XXX see above */
p2mt = p2m_mmio_dm;
}
if ( page_order )
*page_order = PAGE_ORDER_4K;
out:
*t = p2mt;
return mfn;
}
static mfn_t
p2m_pt_get_entry(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
unsigned long l1e_flags;
p2m_type_t l1t;
ASSERT(paging_mode_translate(p2m->domain));
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
*t = p2m_mmio_dm;
/* Not implemented except with EPT */
*a = p2m_access_rwx;
if ( gfn > p2m->max_mapped_pfn )
/* This pfn is higher than the highest the p2m map currently holds */
return _mfn(INVALID_MFN);
mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
{
l4_pgentry_t *l4e = map_domain_page(mfn_x(mfn));
l4e += l4_table_offset(addr);
if ( (l4e_get_flags(*l4e) & _PAGE_PRESENT) == 0 )
{
unmap_domain_page(l4e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l4e_get_pfn(*l4e));
unmap_domain_page(l4e);
}
{
l3_pgentry_t *l3e = map_domain_page(mfn_x(mfn));
l3e += l3_table_offset(addr);
pod_retry_l3:
if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
{
if ( p2m_flags_to_type(l3e_get_flags(*l3e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
}
else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l3e);
return _mfn(INVALID_MFN);
}
else if ( (l3e_get_flags(*l3e) & _PAGE_PSE) )
{
mfn = _mfn(l3e_get_pfn(*l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
*t = p2m_flags_to_type(l3e_get_flags(*l3e));
unmap_domain_page(l3e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_1G;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l3e_get_pfn(*l3e));
unmap_domain_page(l3e);
}
l2e = map_domain_page(mfn_x(mfn));
l2e += l2_table_offset(addr);
pod_retry_l2:
if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )
{
/* PoD: Try to populate a 2-meg chunk */
if ( p2m_flags_to_type(l2e_get_flags(*l2e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_2M, q) )
goto pod_retry_l2;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l2e);
return _mfn(INVALID_MFN);
}
else if ( (l2e_get_flags(*l2e) & _PAGE_PSE) )
{
mfn = _mfn(l2e_get_pfn(*l2e) + l1_table_offset(addr));
*t = p2m_flags_to_type(l2e_get_flags(*l2e));
unmap_domain_page(l2e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_2M;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l2e_get_pfn(*l2e));
unmap_domain_page(l2e);
l1e = map_domain_page(mfn_x(mfn));
l1e += l1_table_offset(addr);
pod_retry_l1:
l1e_flags = l1e_get_flags(*l1e);
l1t = p2m_flags_to_type(l1e_flags);
if ( ((l1e_flags & _PAGE_PRESENT) == 0) && (!p2m_is_paging(l1t)) )
{
/* PoD: Try to populate */
if ( l1t == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_4K, q) )
goto pod_retry_l1;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l1e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l1e_get_pfn(*l1e));
*t = l1t;
unmap_domain_page(l1e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t) || p2m_is_paging(*t));
if ( page_order )
*page_order = PAGE_ORDER_4K;
return (p2m_is_valid(*t) || p2m_is_grant(*t)) ? mfn : _mfn(INVALID_MFN);
}
int
guest_physmap_add_entry(struct domain *d, unsigned long gfn,
unsigned long mfn, unsigned int page_order,
p2m_type_t t)
{
struct p2m_domain *p2m = p2m_get_hostp2m(d);
unsigned long i, ogfn;
p2m_type_t ot;
p2m_access_t a;
mfn_t omfn;
int pod_count = 0;
int rc = 0;
if ( !paging_mode_translate(d) )
{
if ( need_iommu(d) && t == p2m_ram_rw )
{
for ( i = 0; i < (1 << page_order); i++ )
{
rc = iommu_map_page(
d, mfn + i, mfn + i, IOMMUF_readable|IOMMUF_writable);
if ( rc != 0 )
{
while ( i-- > 0 )
iommu_unmap_page(d, mfn + i);
return rc;
}
}
}
return 0;
}
p2m_lock(p2m);
P2M_DEBUG("adding gfn=%#lx mfn=%#lx\n", gfn, mfn);
/* First, remove m->p mappings for existing p->m mappings */
for ( i = 0; i < (1UL << page_order); i++ )
{
omfn = p2m->get_entry(p2m, gfn + i, &ot, &a, 0, NULL);
if ( p2m_is_shared(ot) )
{
/* Do an unshare to cleanly take care of all corner
* cases. */
int rc;
rc = mem_sharing_unshare_page(p2m->domain, gfn + i, 0);
if ( rc )
{
p2m_unlock(p2m);
/* NOTE: Should a guest domain bring this upon itself,
* there is not a whole lot we can do. We are buried
* deep in locks from most code paths by now. So, fail
* the call and don't try to sleep on a wait queue
* while placing the mem event.
*
* However, all current (changeset 3432abcf9380) code
* paths avoid this unsavoury situation. For now.
*
* Foreign domains are okay to place an event as they
* won't go to sleep. */
(void)mem_sharing_notify_enomem(p2m->domain, gfn + i, 0);
return rc;
}
omfn = p2m->get_entry(p2m, gfn + i, &ot, &a, 0, NULL);
ASSERT(!p2m_is_shared(ot));
}
if ( p2m_is_grant(ot) )
{
/* Really shouldn't be unmapping grant maps this way */
domain_crash(d);
p2m_unlock(p2m);
return -EINVAL;
}
else if ( p2m_is_ram(ot) && !p2m_is_paged(ot) )
{
ASSERT(mfn_valid(omfn));
set_gpfn_from_mfn(mfn_x(omfn), INVALID_M2P_ENTRY);
}
else if ( ot == p2m_populate_on_demand )
{
/* Count how man PoD entries we'll be replacing if successful */
pod_count++;
}
else if ( p2m_is_paging(ot) && (ot != p2m_ram_paging_out) )
{
/* We're plugging a hole in the physmap where a paged out page was */
atomic_dec(&d->paged_pages);
}
}
/* Then, look for m->p mappings for this range and deal with them */
for ( i = 0; i < (1UL << page_order); i++ )
{
if ( page_get_owner(mfn_to_page(_mfn(mfn + i))) == dom_cow )
{
/* This is no way to add a shared page to your physmap! */
gdprintk(XENLOG_ERR, "Adding shared mfn %lx directly to dom %hu "
"physmap not allowed.\n", mfn+i, d->domain_id);
p2m_unlock(p2m);
return -EINVAL;
}
if ( page_get_owner(mfn_to_page(_mfn(mfn + i))) != d )
continue;
ogfn = mfn_to_gfn(d, _mfn(mfn+i));
if ( (ogfn != INVALID_M2P_ENTRY) && (ogfn != gfn + i) )
{
/* This machine frame is already mapped at another physical
* address */
P2M_DEBUG("aliased! mfn=%#lx, old gfn=%#lx, new gfn=%#lx\n",
mfn + i, ogfn, gfn + i);
omfn = p2m->get_entry(p2m, ogfn, &ot, &a, 0, NULL);
if ( p2m_is_ram(ot) && !p2m_is_paged(ot) )
{
ASSERT(mfn_valid(omfn));
P2M_DEBUG("old gfn=%#lx -> mfn %#lx\n",
ogfn , mfn_x(omfn));
if ( mfn_x(omfn) == (mfn + i) )
p2m_remove_page(p2m, ogfn, mfn + i, 0);
}
}
}
/* Now, actually do the two-way mapping */
if ( mfn_valid(_mfn(mfn)) )
{
if ( !set_p2m_entry(p2m, gfn, _mfn(mfn), page_order, t, p2m->default_access) )
{
rc = -EINVAL;
goto out; /* Failed to update p2m, bail without updating m2p. */
}
if ( !p2m_is_grant(t) )
{
for ( i = 0; i < (1UL << page_order); i++ )
set_gpfn_from_mfn(mfn+i, gfn+i);
}
}
else
{
gdprintk(XENLOG_WARNING, "Adding bad mfn to p2m map (%#lx -> %#lx)\n",
gfn, mfn);
if ( !set_p2m_entry(p2m, gfn, _mfn(INVALID_MFN), page_order,
p2m_invalid, p2m->default_access) )
rc = -EINVAL;
else
{
pod_lock(p2m);
p2m->pod.entry_count -= pod_count;
BUG_ON(p2m->pod.entry_count < 0);
pod_unlock(p2m);
}
}
out:
p2m_unlock(p2m);
return rc;
}