void dump_pt_walk(lpae_t *first, paddr_t addr)
{
lpae_t *second = NULL, *third = NULL;
if ( first_table_offset(addr) >= LPAE_ENTRIES )
return;
printk("1ST[0x%x] = 0x%"PRIpaddr"\n", first_table_offset(addr),
first[first_table_offset(addr)].bits);
if ( !first[first_table_offset(addr)].walk.valid ||
!first[first_table_offset(addr)].walk.table )
goto done;
second = map_domain_page(first[first_table_offset(addr)].walk.base);
printk("2ND[0x%x] = 0x%"PRIpaddr"\n", second_table_offset(addr),
second[second_table_offset(addr)].bits);
if ( !second[second_table_offset(addr)].walk.valid ||
!second[second_table_offset(addr)].walk.table )
goto done;
third = map_domain_page(second[second_table_offset(addr)].walk.base);
printk("3RD[0x%x] = 0x%"PRIpaddr"\n", third_table_offset(addr),
third[third_table_offset(addr)].bits);
done:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
}
LOCAL int mcd_event_enable(struct domain *d, mfn_t ring_mfn, mfn_t shared_mfn)
{
int rc = 0;
/* Map ring and shared pages */
d->mcd_event.ring_page = map_domain_page(mfn_x(ring_mfn));
if ( d->mcd_event.ring_page == NULL )
goto err;
//printk("domain_id = %d, ring_page = %p \n", d->domain_id, d->mcd_event.ring_page);
// default = 1
d->mcd_event.num_shared_page = 1;
d->mcd_event.shared_page[0] = map_domain_page(mfn_x(shared_mfn));
if ( d->mcd_event.shared_page[0] == NULL )
goto err_ring;
// TODO check this... whether we need this or just using ring for notification...
// TODO however, ring notification should have some delay incurred before receiving...
/* Allocate event channel */
rc = alloc_unbound_xen_event_channel(d->vcpu[0], current->domain->domain_id);
if ( rc < 0 )
goto err_shared;
// XXX since we use data as a buffer.. this is the way to avoid future conflict
memcpy(((mcd_event_shared_page_t *)d->mcd_event.shared_page[0])->data, &rc, sizeof(int));
d->mcd_event.xen_port = rc;
/* Prepare ring buffer */
FRONT_RING_INIT(&d->mcd_event.front_ring,
(mcd_event_sring_t *)d->mcd_event.ring_page,
PAGE_SIZE);
//printk("ring buffer size = %d \n", (&(d->mcd_event.front_ring))->nr_ents);
mcd_event_ring_lock_init(d);
/* Wake any VCPUs paused for memory events */
//mcd_event_unpause_vcpus(d);
init_mcdctl();
return 0;
err_shared:
unmap_domain_page(d->mcd_event.shared_page[0]);
d->mcd_event.shared_page[0] = NULL;
err_ring:
unmap_domain_page(d->mcd_event.ring_page);
d->mcd_event.ring_page = NULL;
err:
return 1;
}
/*
* Lookup the MFN corresponding to a domain's PFN.
*
* There are no processor functions to do a stage 2 only lookup therefore we
* do a a software walk.
*/
paddr_t p2m_lookup(struct domain *d, paddr_t paddr, p2m_type_t *t)
{
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t pte, *first = NULL, *second = NULL, *third = NULL;
paddr_t maddr = INVALID_PADDR;
p2m_type_t _t;
/* Allow t to be NULL */
t = t ?: &_t;
*t = p2m_invalid;
spin_lock(&p2m->lock);
first = p2m_map_first(p2m, paddr);
if ( !first )
goto err;
pte = first[first_table_offset(paddr)];
if ( !pte.p2m.valid || !pte.p2m.table )
goto done;
second = map_domain_page(pte.p2m.base);
pte = second[second_table_offset(paddr)];
if ( !pte.p2m.valid || !pte.p2m.table )
goto done;
third = map_domain_page(pte.p2m.base);
pte = third[third_table_offset(paddr)];
/* This bit must be one in the level 3 entry */
if ( !pte.p2m.table )
pte.bits = 0;
done:
if ( pte.p2m.valid )
{
ASSERT(pte.p2m.type != p2m_invalid);
maddr = (pte.bits & PADDR_MASK & PAGE_MASK) | (paddr & ~PAGE_MASK);
*t = pte.p2m.type;
}
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
err:
spin_unlock(&p2m->lock);
return maddr;
}
static void enable_hypercall_page(void)
{
struct domain *d = current->domain;
unsigned long gmfn = d->arch.hvm_domain.viridian.hypercall_gpa.fields.pfn;
unsigned long mfn = gmfn_to_mfn(d, gmfn);
uint8_t *p;
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
{
gdprintk(XENLOG_WARNING, "Bad GMFN %lx (MFN %lx)\n", gmfn, mfn);
return;
}
p = map_domain_page(mfn);
/*
* We set the bit 31 in %eax (reserved field in the Viridian hypercall
* calling convention) to differentiate Xen and Viridian hypercalls.
*/
*(u8 *)(p + 0) = 0x0d; /* orl $0x80000000, %eax */
*(u32 *)(p + 1) = 0x80000000;
*(u8 *)(p + 5) = 0x0f; /* vmcall/vmmcall */
*(u8 *)(p + 6) = 0x01;
*(u8 *)(p + 7) = ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
? 0xc1 : 0xd9);
*(u8 *)(p + 8) = 0xc3; /* ret */
memset(p + 9, 0xcc, PAGE_SIZE - 9); /* int3, int3, ... */
unmap_domain_page(p);
put_page_and_type(mfn_to_page(mfn));
}
/*
* Mark (via clearing the U flag) as needing P2M type re-calculation all valid
* present entries at the targeted level for the passed in GFN range, which is
* guaranteed to not cross a page (table) boundary at that level.
*/
static int p2m_pt_set_recalc_range(struct p2m_domain *p2m,
unsigned int level,
unsigned long first_gfn,
unsigned long last_gfn)
{
void *table;
unsigned long gfn_remainder = first_gfn, remainder;
unsigned int i;
l1_pgentry_t *pent, *plast;
int err = 0;
table = map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
for ( i = 4; i-- > level; )
{
remainder = gfn_remainder;
pent = p2m_find_entry(table, &remainder, first_gfn,
i * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER);
if ( !pent )
{
err = -EINVAL;
goto out;
}
if ( !(l1e_get_flags(*pent) & _PAGE_PRESENT) )
goto out;
err = p2m_next_level(p2m, &table, &gfn_remainder, first_gfn,
i * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER,
pgt[i - 1], 1);
if ( err )
goto out;
}
remainder = gfn_remainder + (last_gfn - first_gfn);
pent = p2m_find_entry(table, &gfn_remainder, first_gfn,
i * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER);
plast = p2m_find_entry(table, &remainder, last_gfn,
i * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER);
if ( pent && plast )
for ( ; pent <= plast; ++pent )
{
l1_pgentry_t e = *pent;
if ( (l1e_get_flags(e) & _PAGE_PRESENT) && !needs_recalc(l1, e) )
{
set_recalc(l1, e);
p2m->write_p2m_entry(p2m, first_gfn, pent, e, level);
}
first_gfn += 1UL << (i * PAGETABLE_ORDER);
}
else
err = -EIO;
out:
unmap_domain_page(table);
return err;
}
static int mem_event_enable(struct domain *d, mfn_t ring_mfn, mfn_t shared_mfn)
{
int rc;
/* Map ring and shared pages */
d->mem_event.ring_page = map_domain_page(mfn_x(ring_mfn));
if ( d->mem_event.ring_page == NULL )
goto err;
d->mem_event.shared_page = map_domain_page(mfn_x(shared_mfn));
if ( d->mem_event.shared_page == NULL )
goto err_ring;
/* Allocate event channel */
rc = alloc_unbound_xen_event_channel(d->vcpu[0],
current->domain->domain_id);
if ( rc < 0 )
goto err_shared;
((mem_event_shared_page_t *)d->mem_event.shared_page)->port = rc;
d->mem_event.xen_port = rc;
/* Prepare ring buffer */
FRONT_RING_INIT(&d->mem_event.front_ring,
(mem_event_sring_t *)d->mem_event.ring_page,
PAGE_SIZE);
mem_event_ring_lock_init(d);
/* Wake any VCPUs paused for memory events */
mem_event_unpause_vcpus(d);
return 0;
err_shared:
unmap_domain_page(d->mem_event.shared_page);
d->mem_event.shared_page = NULL;
err_ring:
unmap_domain_page(d->mem_event.ring_page);
d->mem_event.ring_page = NULL;
err:
return 1;
}
/*
* Lookup the MFN corresponding to a domain's PFN.
*
* There are no processor functions to do a stage 2 only lookup therefore we
* do a a software walk.
*/
paddr_t p2m_lookup(struct domain *d, paddr_t paddr)
{
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t pte, *first = NULL, *second = NULL, *third = NULL;
paddr_t maddr = INVALID_PADDR;
spin_lock(&p2m->lock);
first = __map_domain_page(p2m->first_level);
pte = first[first_table_offset(paddr)];
if ( !pte.p2m.valid || !pte.p2m.table )
goto done;
second = map_domain_page(pte.p2m.base);
pte = second[second_table_offset(paddr)];
if ( !pte.p2m.valid || !pte.p2m.table )
goto done;
third = map_domain_page(pte.p2m.base);
pte = third[third_table_offset(paddr)];
/* This bit must be one in the level 3 entry */
if ( !pte.p2m.table )
pte.bits = 0;
done:
if ( pte.p2m.valid )
maddr = (pte.bits & PADDR_MASK & PAGE_MASK) | (paddr & ~PAGE_MASK);
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
spin_unlock(&p2m->lock);
return maddr;
}
void flush_page_to_ram(unsigned long mfn)
{
void *v = map_domain_page(_mfn(mfn));
clean_and_invalidate_dcache_va_range(v, PAGE_SIZE);
unmap_domain_page(v);
/*
* For some of the instruction cache (such as VIPT), the entire I-Cache
* needs to be flushed to guarantee that all the aliases of a given
* physical address will be removed from the cache.
* Invalidating the I-Cache by VA highly depends on the behavior of the
* I-Cache (See D4.9.2 in ARM DDI 0487A.k_iss10775). Instead of using flush
* by VA on select platforms, we just flush the entire cache here.
*/
invalidate_icache();
}
LOCAL int mcd_event_add_shared_page(struct domain *d, mfn_t shared_mfn)
{
int curr_num = d->mcd_event.num_shared_page;
if ( curr_num < 0 || curr_num >= MAX_SHARED_PAGES ) {
printk("curr_num(%d) is wrong..\n", curr_num);
return 1;
}
d->mcd_event.shared_page[curr_num] = map_domain_page(mfn_x(shared_mfn));
if ( d->mcd_event.shared_page[curr_num] == NULL ) {
printk("d->mcd_event.shared_page[%d] == NULL.....\n", curr_num);
return 1;
}
d->mcd_event.num_shared_page++;
return 0;
}
/* Free intermediate tables from a p2m sub-tree */
static void
p2m_free_entry(struct p2m_domain *p2m, l1_pgentry_t *p2m_entry, int page_order)
{
/* End if the entry is a leaf entry. */
if ( page_order == PAGE_ORDER_4K
|| !(l1e_get_flags(*p2m_entry) & _PAGE_PRESENT)
|| (l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
return;
if ( page_order > PAGE_ORDER_2M )
{
l1_pgentry_t *l3_table = map_domain_page(_mfn(l1e_get_pfn(*p2m_entry)));
for ( int i = 0; i < L3_PAGETABLE_ENTRIES; i++ )
p2m_free_entry(p2m, l3_table + i, page_order - 9);
unmap_domain_page(l3_table);
}
p2m_free_ptp(p2m, mfn_to_page(_mfn(l1e_get_pfn(*p2m_entry))));
}
static u64 iommu_l2e_from_pfn(struct page_info *table, int level,
unsigned long io_pfn)
{
unsigned long offset;
void *pde = NULL;
void *table_vaddr;
u64 next_table_maddr = 0;
BUG_ON( table == NULL || level == 0 );
while ( level > 1 )
{
offset = io_pfn >> ((PTE_PER_TABLE_SHIFT *
(level - IOMMU_PAGING_MODE_LEVEL_1)));
offset &= ~PTE_PER_TABLE_MASK;
table_vaddr = map_domain_page(page_to_mfn(table));
pde = table_vaddr + (offset * IOMMU_PAGE_TABLE_ENTRY_SIZE);
next_table_maddr = amd_iommu_get_next_table_from_pte(pde);
if ( !amd_iommu_is_pte_present(pde) )
{
if ( next_table_maddr == 0 )
{
table = alloc_amd_iommu_pgtable();
if ( table == NULL )
return 0;
next_table_maddr = page_to_maddr(table);
amd_iommu_set_page_directory_entry(
(u32 *)pde, next_table_maddr, level - 1);
}
else /* should never reach here */
return 0;
}
unmap_domain_page(table_vaddr);
table = maddr_to_page(next_table_maddr);
level--;
}
return next_table_maddr;
}
/* Returns: mfn for the given (pv guest) vaddr */
static unsigned long
dbg_pv_va2mfn(dbgva_t vaddr, struct domain *dp, uint64_t pgd3val)
{
l3_pgentry_t l3e, *l3t;
l2_pgentry_t l2e, *l2t;
l1_pgentry_t l1e, *l1t;
unsigned long cr3 = (pgd3val ? pgd3val : dp->vcpu[0]->arch.cr3);
unsigned long mfn = cr3 >> PAGE_SHIFT;
DBGP2("vaddr:%lx domid:%d cr3:%lx pgd3:%lx\n", vaddr, dp->domain_id,
cr3, pgd3val);
if ( pgd3val == 0 )
{
l3t = map_domain_page(mfn);
l3t += (cr3 & 0xFE0UL) >> 3;
l3e = l3t[l3_table_offset(vaddr)];
mfn = l3e_get_pfn(l3e);
unmap_domain_page(l3t);
if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
return INVALID_MFN;
}
unsigned long hap_gva_to_gfn(GUEST_PAGING_LEVELS)(
struct vcpu *v, unsigned long gva, uint32_t *pfec)
{
unsigned long cr3;
uint32_t missing;
mfn_t top_mfn;
void *top_map;
p2m_type_t p2mt;
walk_t gw;
/* Get the top-level table's MFN */
cr3 = v->arch.hvm_vcpu.guest_cr[3];
top_mfn = gfn_to_mfn(v->domain, _gfn(cr3 >> PAGE_SHIFT), &p2mt);
if ( !p2m_is_ram(p2mt) )
{
pfec[0] &= ~PFEC_page_present;
return INVALID_GFN;
}
/* Map the top-level table and call the tree-walker */
ASSERT(mfn_valid(mfn_x(top_mfn)));
top_map = map_domain_page(mfn_x(top_mfn));
#if GUEST_PAGING_LEVELS == 3
top_map += (cr3 & ~(PAGE_MASK | 31));
#endif
missing = guest_walk_tables(v, gva, &gw, pfec[0], top_mfn, top_map);
unmap_domain_page(top_map);
/* Interpret the answer */
if ( missing == 0 )
return gfn_x(guest_l1e_get_gfn(gw.l1e));
if ( missing & _PAGE_PRESENT )
pfec[0] &= ~PFEC_page_present;
return INVALID_GFN;
}
static int setup_compat_l4(struct vcpu *v)
{
struct page_info *pg;
l4_pgentry_t *l4tab;
mfn_t mfn;
pg = alloc_domheap_page(v->domain, MEMF_no_owner);
if ( pg == NULL )
return -ENOMEM;
mfn = page_to_mfn(pg);
l4tab = map_domain_page(mfn);
clear_page(l4tab);
init_xen_l4_slots(l4tab, mfn, v->domain, INVALID_MFN, false);
unmap_domain_page(l4tab);
/* This page needs to look like a pagetable so that it can be shadowed */
pg->u.inuse.type_info = PGT_l4_page_table | PGT_validated | 1;
v->arch.guest_table = pagetable_from_page(pg);
v->arch.guest_table_user = v->arch.guest_table;
return 0;
}
/* Returns: 0 for success, -errno for failure */
static int
p2m_next_level(struct p2m_domain *p2m, void **table,
unsigned long *gfn_remainder, unsigned long gfn, u32 shift,
u32 max, unsigned long type, bool_t unmap)
{
l1_pgentry_t *l1_entry;
l1_pgentry_t *p2m_entry;
l1_pgentry_t new_entry;
void *next;
int i;
if ( !(p2m_entry = p2m_find_entry(*table, gfn_remainder, gfn,
shift, max)) )
return -ENOENT;
/* PoD/paging: Not present doesn't imply empty. */
if ( !l1e_get_flags(*p2m_entry) )
{
struct page_info *pg;
pg = p2m_alloc_ptp(p2m, type);
if ( pg == NULL )
return -ENOMEM;
new_entry = l1e_from_pfn(mfn_x(page_to_mfn(pg)),
P2M_BASE_FLAGS | _PAGE_RW);
switch ( type ) {
case PGT_l3_page_table:
p2m_add_iommu_flags(&new_entry, 3, IOMMUF_readable|IOMMUF_writable);
p2m->write_p2m_entry(p2m, gfn, p2m_entry, new_entry, 4);
break;
case PGT_l2_page_table:
p2m_add_iommu_flags(&new_entry, 2, IOMMUF_readable|IOMMUF_writable);
p2m->write_p2m_entry(p2m, gfn, p2m_entry, new_entry, 3);
break;
case PGT_l1_page_table:
p2m_add_iommu_flags(&new_entry, 1, IOMMUF_readable|IOMMUF_writable);
p2m->write_p2m_entry(p2m, gfn, p2m_entry, new_entry, 2);
break;
default:
BUG();
break;
}
}
ASSERT(l1e_get_flags(*p2m_entry) & (_PAGE_PRESENT|_PAGE_PSE));
/* split 1GB pages into 2MB pages */
if ( type == PGT_l2_page_table && (l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
unsigned long flags, pfn;
struct page_info *pg;
pg = p2m_alloc_ptp(p2m, PGT_l2_page_table);
if ( pg == NULL )
return -ENOMEM;
flags = l1e_get_flags(*p2m_entry);
pfn = l1e_get_pfn(*p2m_entry);
l1_entry = __map_domain_page(pg);
for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
{
new_entry = l1e_from_pfn(pfn + (i * L1_PAGETABLE_ENTRIES), flags);
p2m_add_iommu_flags(&new_entry, 1, IOMMUF_readable|IOMMUF_writable);
p2m->write_p2m_entry(p2m, gfn, l1_entry + i, new_entry, 2);
}
unmap_domain_page(l1_entry);
new_entry = l1e_from_pfn(mfn_x(page_to_mfn(pg)),
P2M_BASE_FLAGS | _PAGE_RW); /* disable PSE */
p2m_add_iommu_flags(&new_entry, 2, IOMMUF_readable|IOMMUF_writable);
p2m->write_p2m_entry(p2m, gfn, p2m_entry, new_entry, 3);
}
/* split single 2MB large page into 4KB page in P2M table */
if ( type == PGT_l1_page_table && (l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
unsigned long flags, pfn;
struct page_info *pg;
pg = p2m_alloc_ptp(p2m, PGT_l1_page_table);
if ( pg == NULL )
return -ENOMEM;
/* New splintered mappings inherit the flags of the old superpage,
* with a little reorganisation for the _PAGE_PSE_PAT bit. */
flags = l1e_get_flags(*p2m_entry);
pfn = l1e_get_pfn(*p2m_entry);
if ( pfn & 1 ) /* ==> _PAGE_PSE_PAT was set */
pfn -= 1; /* Clear it; _PAGE_PSE becomes _PAGE_PAT */
else
flags &= ~_PAGE_PSE; /* Clear _PAGE_PSE (== _PAGE_PAT) */
l1_entry = __map_domain_page(pg);
for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
{
new_entry = l1e_from_pfn(pfn + i, flags);
p2m_add_iommu_flags(&new_entry, 0, 0);
p2m->write_p2m_entry(p2m, gfn, l1_entry + i, new_entry, 1);
}
unmap_domain_page(l1_entry);
new_entry = l1e_from_pfn(mfn_x(page_to_mfn(pg)),
P2M_BASE_FLAGS | _PAGE_RW);
p2m_add_iommu_flags(&new_entry, 1, IOMMUF_readable|IOMMUF_writable);
p2m->write_p2m_entry(p2m, gfn, p2m_entry, new_entry, 2);
}
next = map_domain_page(_mfn(l1e_get_pfn(*p2m_entry)));
if ( unmap )
unmap_domain_page(*table);
*table = next;
return 0;
}
DBGP2("vaddr:%lx domid:%d cr3:%lx pgd3:%lx\n", vaddr, dp->domain_id,
cr3, pgd3val);
if ( pgd3val == 0 )
{
l3t = map_domain_page(mfn);
l3t += (cr3 & 0xFE0UL) >> 3;
l3e = l3t[l3_table_offset(vaddr)];
mfn = l3e_get_pfn(l3e);
unmap_domain_page(l3t);
if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
return INVALID_MFN;
}
l2t = map_domain_page(mfn);
l2e = l2t[l2_table_offset(vaddr)];
mfn = l2e_get_pfn(l2e);
unmap_domain_page(l2t);
if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) ||
(l2e_get_flags(l2e) & _PAGE_PSE) )
return INVALID_MFN;
l1t = map_domain_page(mfn);
l1e = l1t[l1_table_offset(vaddr)];
mfn = l1e_get_pfn(l1e);
unmap_domain_page(l1t);
return mfn_valid(mfn) ? mfn : INVALID_MFN;
}
#endif /* defined(__x86_64__) */
static mfn_t
p2m_pt_get_entry(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
unsigned long l1e_flags;
p2m_type_t l1t;
ASSERT(paging_mode_translate(p2m->domain));
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
*t = p2m_mmio_dm;
/* Not implemented except with EPT */
*a = p2m_access_rwx;
if ( gfn > p2m->max_mapped_pfn )
/* This pfn is higher than the highest the p2m map currently holds */
return _mfn(INVALID_MFN);
mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
{
l4_pgentry_t *l4e = map_domain_page(mfn_x(mfn));
l4e += l4_table_offset(addr);
if ( (l4e_get_flags(*l4e) & _PAGE_PRESENT) == 0 )
{
unmap_domain_page(l4e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l4e_get_pfn(*l4e));
unmap_domain_page(l4e);
}
{
l3_pgentry_t *l3e = map_domain_page(mfn_x(mfn));
l3e += l3_table_offset(addr);
pod_retry_l3:
if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
{
if ( p2m_flags_to_type(l3e_get_flags(*l3e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
}
else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l3e);
return _mfn(INVALID_MFN);
}
else if ( (l3e_get_flags(*l3e) & _PAGE_PSE) )
{
mfn = _mfn(l3e_get_pfn(*l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
*t = p2m_flags_to_type(l3e_get_flags(*l3e));
unmap_domain_page(l3e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_1G;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l3e_get_pfn(*l3e));
unmap_domain_page(l3e);
}
l2e = map_domain_page(mfn_x(mfn));
l2e += l2_table_offset(addr);
pod_retry_l2:
if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )
{
/* PoD: Try to populate a 2-meg chunk */
if ( p2m_flags_to_type(l2e_get_flags(*l2e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_2M, q) )
goto pod_retry_l2;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l2e);
return _mfn(INVALID_MFN);
}
else if ( (l2e_get_flags(*l2e) & _PAGE_PSE) )
{
mfn = _mfn(l2e_get_pfn(*l2e) + l1_table_offset(addr));
*t = p2m_flags_to_type(l2e_get_flags(*l2e));
unmap_domain_page(l2e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_2M;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l2e_get_pfn(*l2e));
unmap_domain_page(l2e);
l1e = map_domain_page(mfn_x(mfn));
l1e += l1_table_offset(addr);
pod_retry_l1:
l1e_flags = l1e_get_flags(*l1e);
l1t = p2m_flags_to_type(l1e_flags);
if ( ((l1e_flags & _PAGE_PRESENT) == 0) && (!p2m_is_paging(l1t)) )
{
/* PoD: Try to populate */
if ( l1t == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_4K, q) )
goto pod_retry_l1;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l1e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l1e_get_pfn(*l1e));
*t = l1t;
unmap_domain_page(l1e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t) || p2m_is_paging(*t));
if ( page_order )
*page_order = PAGE_ORDER_4K;
return (p2m_is_valid(*t) || p2m_is_grant(*t)) ? mfn : _mfn(INVALID_MFN);
}
/*
* Handle possibly necessary P2M type re-calculation (U flag clear for a
* present entry) for the entries in the page table hierarchy for the given
* GFN. Propagate the re-calculation flag down to the next page table level
* for entries not involved in the translation of the given GFN.
*/
static int do_recalc(struct p2m_domain *p2m, unsigned long gfn)
{
void *table;
unsigned long gfn_remainder = gfn;
unsigned int level = 4;
l1_pgentry_t *pent;
int err = 0;
table = map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
while ( --level )
{
unsigned long remainder = gfn_remainder;
pent = p2m_find_entry(table, &remainder, gfn,
level * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER);
if ( !pent || !(l1e_get_flags(*pent) & _PAGE_PRESENT) )
goto out;
if ( l1e_get_flags(*pent) & _PAGE_PSE )
{
unsigned long mask = ~0UL << (level * PAGETABLE_ORDER);
if ( !needs_recalc(l1, *pent) ||
!p2m_is_changeable(p2m_flags_to_type(l1e_get_flags(*pent))) ||
p2m_is_logdirty_range(p2m, gfn & mask, gfn | ~mask) >= 0 )
break;
}
err = p2m_next_level(p2m, &table, &gfn_remainder, gfn,
level * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER,
pgt[level - 1], 0);
if ( err )
goto out;
if ( needs_recalc(l1, *pent) )
{
l1_pgentry_t e = *pent, *ptab = table;
unsigned int i;
if ( !valid_recalc(l1, e) )
P2M_DEBUG("bogus recalc state at d%d:%lx:%u\n",
p2m->domain->domain_id, gfn, level);
remainder = gfn_remainder;
for ( i = 0; i < (1 << PAGETABLE_ORDER); ++i )
{
l1_pgentry_t ent = ptab[i];
if ( (l1e_get_flags(ent) & _PAGE_PRESENT) &&
!needs_recalc(l1, ent) )
{
set_recalc(l1, ent);
p2m->write_p2m_entry(p2m, gfn - remainder, &ptab[i],
ent, level);
}
remainder -= 1UL << ((level - 1) * PAGETABLE_ORDER);
}
smp_wmb();
clear_recalc(l1, e);
p2m->write_p2m_entry(p2m, gfn, pent, e, level + 1);
}
unmap_domain_page((void *)((unsigned long)pent & PAGE_MASK));
}
pent = p2m_find_entry(table, &gfn_remainder, gfn,
level * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER);
if ( pent && (l1e_get_flags(*pent) & _PAGE_PRESENT) &&
needs_recalc(l1, *pent) )
{
l1_pgentry_t e = *pent;
if ( !valid_recalc(l1, e) )
P2M_DEBUG("bogus recalc leaf at d%d:%lx:%u\n",
p2m->domain->domain_id, gfn, level);
if ( p2m_is_changeable(p2m_flags_to_type(l1e_get_flags(e))) )
{
unsigned long mask = ~0UL << (level * PAGETABLE_ORDER);
p2m_type_t p2mt = p2m_is_logdirty_range(p2m, gfn & mask, gfn | ~mask)
? p2m_ram_logdirty : p2m_ram_rw;
unsigned long mfn = l1e_get_pfn(e);
unsigned long flags = p2m_type_to_flags(p2mt, _mfn(mfn));
if ( level )
{
if ( flags & _PAGE_PAT )
{
BUILD_BUG_ON(_PAGE_PAT != _PAGE_PSE);
mfn |= _PAGE_PSE_PAT >> PAGE_SHIFT;
}
else
mfn &= ~(_PAGE_PSE_PAT >> PAGE_SHIFT);
flags |= _PAGE_PSE;
}
long p2m_pt_audit_p2m(struct p2m_domain *p2m)
{
unsigned long entry_count = 0, pmbad = 0;
unsigned long mfn, gfn, m2pfn;
int test_linear;
struct domain *d = p2m->domain;
ASSERT(p2m_locked_by_me(p2m));
ASSERT(pod_locked_by_me(p2m));
test_linear = ( (d == current->domain)
&& !pagetable_is_null(current->arch.monitor_table) );
if ( test_linear )
flush_tlb_local();
/* Audit part one: walk the domain's p2m table, checking the entries. */
if ( pagetable_get_pfn(p2m_get_pagetable(p2m)) != 0 )
{
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
int i1, i2;
#if CONFIG_PAGING_LEVELS == 4
l4_pgentry_t *l4e;
l3_pgentry_t *l3e;
int i4, i3;
l4e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#else /* CONFIG_PAGING_LEVELS == 3 */
l3_pgentry_t *l3e;
int i3;
l3e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#endif
gfn = 0;
#if CONFIG_PAGING_LEVELS >= 4
for ( i4 = 0; i4 < L4_PAGETABLE_ENTRIES; i4++ )
{
if ( !(l4e_get_flags(l4e[i4]) & _PAGE_PRESENT) )
{
gfn += 1 << (L4_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
l3e = map_domain_page(mfn_x(_mfn(l4e_get_pfn(l4e[i4]))));
#endif
for ( i3 = 0;
i3 < ((CONFIG_PAGING_LEVELS==4) ? L3_PAGETABLE_ENTRIES : 8);
i3++ )
{
if ( !(l3e_get_flags(l3e[i3]) & _PAGE_PRESENT) )
{
gfn += 1 << (L3_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
/* check for 1GB super page */
if ( l3e_get_flags(l3e[i3]) & _PAGE_PSE )
{
mfn = l3e_get_pfn(l3e[i3]);
ASSERT(mfn_valid(_mfn(mfn)));
/* we have to cover 512x512 4K pages */
for ( i2 = 0;
i2 < (L2_PAGETABLE_ENTRIES * L1_PAGETABLE_ENTRIES);
i2++)
{
m2pfn = get_gpfn_from_mfn(mfn+i2);
if ( m2pfn != (gfn + i2) )
{
pmbad++;
P2M_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn+i2, mfn+i2,
m2pfn);
BUG();
}
gfn += 1 << (L3_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
}
l2e = map_domain_page(mfn_x(_mfn(l3e_get_pfn(l3e[i3]))));
for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; i2++ )
{
if ( !(l2e_get_flags(l2e[i2]) & _PAGE_PRESENT) )
{
if ( (l2e_get_flags(l2e[i2]) & _PAGE_PSE)
&& ( p2m_flags_to_type(l2e_get_flags(l2e[i2]))
== p2m_populate_on_demand ) )
entry_count+=SUPERPAGE_PAGES;
gfn += 1 << (L2_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
/* check for super page */
if ( l2e_get_flags(l2e[i2]) & _PAGE_PSE )
{
mfn = l2e_get_pfn(l2e[i2]);
ASSERT(mfn_valid(_mfn(mfn)));
for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++)
{
m2pfn = get_gpfn_from_mfn(mfn+i1);
/* Allow shared M2Ps */
if ( (m2pfn != (gfn + i1)) &&
(m2pfn != SHARED_M2P_ENTRY) )
{
pmbad++;
P2M_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn+i1, mfn+i1,
m2pfn);
BUG();
}
}
gfn += 1 << (L2_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
l1e = map_domain_page(mfn_x(_mfn(l2e_get_pfn(l2e[i2]))));
for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++, gfn++ )
{
p2m_type_t type;
type = p2m_flags_to_type(l1e_get_flags(l1e[i1]));
if ( !(l1e_get_flags(l1e[i1]) & _PAGE_PRESENT) )
{
if ( type == p2m_populate_on_demand )
entry_count++;
continue;
}
mfn = l1e_get_pfn(l1e[i1]);
ASSERT(mfn_valid(_mfn(mfn)));
m2pfn = get_gpfn_from_mfn(mfn);
if ( m2pfn != gfn &&
type != p2m_mmio_direct &&
!p2m_is_grant(type) &&
!p2m_is_shared(type) )
{
pmbad++;
printk("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn, mfn, m2pfn);
P2M_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn, mfn, m2pfn);
BUG();
}
}
unmap_domain_page(l1e);
}
unmap_domain_page(l2e);
}
#if CONFIG_PAGING_LEVELS >= 4
unmap_domain_page(l3e);
}
#endif
#if CONFIG_PAGING_LEVELS == 4
unmap_domain_page(l4e);
#else /* CONFIG_PAGING_LEVELS == 3 */
unmap_domain_page(l3e);
#endif
}
if ( entry_count != p2m->pod.entry_count )
{
printk("%s: refcounted entry count %ld, audit count %lu!\n",
__func__,
p2m->pod.entry_count,
entry_count);
BUG();
}
return pmbad;
}
/* Walk the whole p2m table, changing any entries of the old type
* to the new type. This is used in hardware-assisted paging to
* quickly enable or diable log-dirty tracking */
static void p2m_change_type_global(struct p2m_domain *p2m,
p2m_type_t ot, p2m_type_t nt)
{
unsigned long mfn, gfn, flags;
l1_pgentry_t l1e_content;
l1_pgentry_t *l1e;
l2_pgentry_t *l2e;
mfn_t l1mfn, l2mfn, l3mfn;
unsigned long i1, i2, i3;
l3_pgentry_t *l3e;
#if CONFIG_PAGING_LEVELS == 4
l4_pgentry_t *l4e;
unsigned long i4;
#endif /* CONFIG_PAGING_LEVELS == 4 */
BUG_ON(p2m_is_grant(ot) || p2m_is_grant(nt));
BUG_ON(ot != nt && (ot == p2m_mmio_direct || nt == p2m_mmio_direct));
if ( !paging_mode_translate(p2m->domain) )
return;
if ( pagetable_get_pfn(p2m_get_pagetable(p2m)) == 0 )
return;
ASSERT(p2m_locked_by_me(p2m));
#if CONFIG_PAGING_LEVELS == 4
l4e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#else /* CONFIG_PAGING_LEVELS == 3 */
l3mfn = _mfn(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
l3e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#endif
#if CONFIG_PAGING_LEVELS >= 4
for ( i4 = 0; i4 < L4_PAGETABLE_ENTRIES; i4++ )
{
if ( !(l4e_get_flags(l4e[i4]) & _PAGE_PRESENT) )
{
continue;
}
l3mfn = _mfn(l4e_get_pfn(l4e[i4]));
l3e = map_domain_page(l4e_get_pfn(l4e[i4]));
#endif
for ( i3 = 0;
i3 < ((CONFIG_PAGING_LEVELS==4) ? L3_PAGETABLE_ENTRIES : 8);
i3++ )
{
if ( !(l3e_get_flags(l3e[i3]) & _PAGE_PRESENT) )
{
continue;
}
if ( (l3e_get_flags(l3e[i3]) & _PAGE_PSE) )
{
flags = l3e_get_flags(l3e[i3]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l3e_get_pfn(l3e[i3]);
gfn = get_gpfn_from_mfn(mfn);
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = l1e_from_pfn(mfn, flags | _PAGE_PSE);
p2m->write_p2m_entry(p2m, gfn,
(l1_pgentry_t *)&l3e[i3],
l3mfn, l1e_content, 3);
continue;
}
l2mfn = _mfn(l3e_get_pfn(l3e[i3]));
l2e = map_domain_page(l3e_get_pfn(l3e[i3]));
for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; i2++ )
{
if ( !(l2e_get_flags(l2e[i2]) & _PAGE_PRESENT) )
{
continue;
}
if ( (l2e_get_flags(l2e[i2]) & _PAGE_PSE) )
{
flags = l2e_get_flags(l2e[i2]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l2e_get_pfn(l2e[i2]);
/* Do not use get_gpfn_from_mfn because it may return
SHARED_M2P_ENTRY */
gfn = (i2 + (i3
#if CONFIG_PAGING_LEVELS >= 4
+ (i4 * L3_PAGETABLE_ENTRIES)
#endif
)
* L2_PAGETABLE_ENTRIES) * L1_PAGETABLE_ENTRIES;
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = l1e_from_pfn(mfn, flags | _PAGE_PSE);
p2m->write_p2m_entry(p2m, gfn,
(l1_pgentry_t *)&l2e[i2],
l2mfn, l1e_content, 2);
continue;
}
l1mfn = _mfn(l2e_get_pfn(l2e[i2]));
l1e = map_domain_page(mfn_x(l1mfn));
for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++, gfn++ )
{
flags = l1e_get_flags(l1e[i1]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l1e_get_pfn(l1e[i1]);
gfn = i1 + (i2 + (i3
#if CONFIG_PAGING_LEVELS >= 4
+ (i4 * L3_PAGETABLE_ENTRIES)
#endif
)
* L2_PAGETABLE_ENTRIES) * L1_PAGETABLE_ENTRIES;
/* create a new 1le entry with the new type */
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = p2m_l1e_from_pfn(mfn, flags);
p2m->write_p2m_entry(p2m, gfn, &l1e[i1],
l1mfn, l1e_content, 1);
}
unmap_domain_page(l1e);
}
unmap_domain_page(l2e);
}
#if CONFIG_PAGING_LEVELS >= 4
unmap_domain_page(l3e);
}
#endif
#if CONFIG_PAGING_LEVELS == 4
unmap_domain_page(l4e);
#else /* CONFIG_PAGING_LEVELS == 3 */
unmap_domain_page(l3e);
#endif
}
static mfn_t
p2m_gfn_to_mfn(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
unsigned long l1e_flags;
p2m_type_t l1t;
ASSERT(paging_mode_translate(p2m->domain));
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
*t = p2m_mmio_dm;
/* Not implemented except with EPT */
*a = p2m_access_rwx;
if ( gfn > p2m->max_mapped_pfn )
/* This pfn is higher than the highest the p2m map currently holds */
return _mfn(INVALID_MFN);
/* Use the fast path with the linear mapping if we can */
if ( p2m == p2m_get_hostp2m(current->domain) )
return p2m_gfn_to_mfn_current(p2m, gfn, t, a, q, page_order);
mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
#if CONFIG_PAGING_LEVELS >= 4
{
l4_pgentry_t *l4e = map_domain_page(mfn_x(mfn));
l4e += l4_table_offset(addr);
if ( (l4e_get_flags(*l4e) & _PAGE_PRESENT) == 0 )
{
unmap_domain_page(l4e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l4e_get_pfn(*l4e));
unmap_domain_page(l4e);
}
#endif
{
l3_pgentry_t *l3e = map_domain_page(mfn_x(mfn));
#if CONFIG_PAGING_LEVELS == 3
/* On PAE hosts the p2m has eight l3 entries, not four (see
* shadow_set_p2m_entry()) so we can't use l3_table_offset.
* Instead, just count the number of l3es from zero. It's safe
* to do this because we already checked that the gfn is within
* the bounds of the p2m. */
l3e += (addr >> L3_PAGETABLE_SHIFT);
#else
l3e += l3_table_offset(addr);
#endif
pod_retry_l3:
if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
{
if ( p2m_flags_to_type(l3e_get_flags(*l3e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
}
else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l3e);
return _mfn(INVALID_MFN);
}
else if ( (l3e_get_flags(*l3e) & _PAGE_PSE) )
{
mfn = _mfn(l3e_get_pfn(*l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
*t = p2m_flags_to_type(l3e_get_flags(*l3e));
unmap_domain_page(l3e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_1G;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l3e_get_pfn(*l3e));
unmap_domain_page(l3e);
}
l2e = map_domain_page(mfn_x(mfn));
l2e += l2_table_offset(addr);
pod_retry_l2:
if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )
{
/* PoD: Try to populate a 2-meg chunk */
if ( p2m_flags_to_type(l2e_get_flags(*l2e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_2M, q) )
goto pod_retry_l2;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l2e);
return _mfn(INVALID_MFN);
}
else if ( (l2e_get_flags(*l2e) & _PAGE_PSE) )
{
mfn = _mfn(l2e_get_pfn(*l2e) + l1_table_offset(addr));
*t = p2m_flags_to_type(l2e_get_flags(*l2e));
unmap_domain_page(l2e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_2M;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l2e_get_pfn(*l2e));
unmap_domain_page(l2e);
l1e = map_domain_page(mfn_x(mfn));
l1e += l1_table_offset(addr);
pod_retry_l1:
l1e_flags = l1e_get_flags(*l1e);
l1t = p2m_flags_to_type(l1e_flags);
if ( ((l1e_flags & _PAGE_PRESENT) == 0) && (!p2m_is_paging(l1t)) )
{
/* PoD: Try to populate */
if ( l1t == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_4K, q) )
goto pod_retry_l1;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l1e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l1e_get_pfn(*l1e));
*t = l1t;
unmap_domain_page(l1e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t) || p2m_is_paging(*t));
if ( page_order )
*page_order = PAGE_ORDER_4K;
return (p2m_is_valid(*t) || p2m_is_grant(*t)) ? mfn : _mfn(INVALID_MFN);
}
// Returns 0 on error (out of memory)
static int
p2m_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
{
// XXX -- this might be able to be faster iff current->domain == d
mfn_t table_mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
void *table =map_domain_page(mfn_x(table_mfn));
unsigned long i, gfn_remainder = gfn;
l1_pgentry_t *p2m_entry;
l1_pgentry_t entry_content;
l2_pgentry_t l2e_content;
l3_pgentry_t l3e_content;
int rv=0;
unsigned int iommu_pte_flags = (p2mt == p2m_ram_rw) ?
IOMMUF_readable|IOMMUF_writable:
0;
unsigned long old_mfn = 0;
if ( tb_init_done )
{
struct {
u64 gfn, mfn;
int p2mt;
int d:16,order:16;
} t;
t.gfn = gfn;
t.mfn = mfn_x(mfn);
t.p2mt = p2mt;
t.d = p2m->domain->domain_id;
t.order = page_order;
__trace_var(TRC_MEM_SET_P2M_ENTRY, 0, sizeof(t), &t);
}
#if CONFIG_PAGING_LEVELS >= 4
if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L4_PAGETABLE_SHIFT - PAGE_SHIFT,
L4_PAGETABLE_ENTRIES, PGT_l3_page_table) )
goto out;
#endif
/*
* Try to allocate 1GB page table if this feature is supported.
*/
if ( page_order == PAGE_ORDER_1G )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
l3e_content = mfn_valid(mfn)
? l3e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) | _PAGE_PSE)
: l3e_empty();
entry_content.l1 = l3e_content.l3;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 3);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/*
* When using PAE Xen, we only allow 33 bits of pseudo-physical
* address in translated guests (i.e. 8 GBytes). This restriction
* comes from wanting to map the P2M table into the 16MB RO_MPT hole
* in Xen's address space for translated PV guests.
* When using AMD's NPT on PAE Xen, we are restricted to 4GB.
*/
else if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
((CONFIG_PAGING_LEVELS == 3)
? (hap_enabled(p2m->domain) ? 4 : 8)
: L3_PAGETABLE_ENTRIES),
PGT_l2_page_table) )
goto out;
if ( page_order == PAGE_ORDER_4K )
{
if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES, PGT_l1_page_table) )
goto out;
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
0, L1_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( mfn_valid(mfn) || (p2mt == p2m_mmio_direct)
|| p2m_is_paging(p2mt) )
entry_content = p2m_l1e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn));
else
entry_content = l1e_empty();
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
/* level 1 entry */
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 1);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
}
else if ( page_order == PAGE_ORDER_2M )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
/* FIXME: Deal with 4k replaced by 2meg pages */
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
if ( mfn_valid(mfn) || p2m_is_magic(p2mt) )
l2e_content = l2e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) |
_PAGE_PSE);
else
l2e_content = l2e_empty();
entry_content.l1 = l2e_content.l2;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 2);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/* Track the highest gfn for which we have ever had a valid mapping */
if ( p2mt != p2m_invalid
&& (gfn + (1UL << page_order) - 1 > p2m->max_mapped_pfn) )
p2m->max_mapped_pfn = gfn + (1UL << page_order) - 1;
if ( iommu_enabled && need_iommu(p2m->domain) )
{
if ( iommu_hap_pt_share )
{
if ( old_mfn && (old_mfn != mfn_x(mfn)) )
amd_iommu_flush_pages(p2m->domain, gfn, page_order);
}
else
{
if ( p2mt == p2m_ram_rw )
for ( i = 0; i < (1UL << page_order); i++ )
iommu_map_page(p2m->domain, gfn+i, mfn_x(mfn)+i,
IOMMUF_readable|IOMMUF_writable);
else
for ( int i = 0; i < (1UL << page_order); i++ )
iommu_unmap_page(p2m->domain, gfn+i);
}
}
/* Success */
rv = 1;
out:
unmap_domain_page(table);
return rv;
}
/*
* Lookup the MFN corresponding to a domain's PFN.
*
* There are no processor functions to do a stage 2 only lookup therefore we
* do a a software walk.
*/
static paddr_t __p2m_lookup(struct domain *d, paddr_t paddr, p2m_type_t *t)
{
struct p2m_domain *p2m = &d->arch.p2m;
const unsigned int offsets[4] = {
zeroeth_table_offset(paddr),
first_table_offset(paddr),
second_table_offset(paddr),
third_table_offset(paddr)
};
const paddr_t masks[4] = {
ZEROETH_MASK, FIRST_MASK, SECOND_MASK, THIRD_MASK
};
lpae_t pte, *map;
paddr_t maddr = INVALID_PADDR;
paddr_t mask = 0;
p2m_type_t _t;
unsigned int level, root_table;
ASSERT(spin_is_locked(&p2m->lock));
BUILD_BUG_ON(THIRD_MASK != PAGE_MASK);
/* Allow t to be NULL */
t = t ?: &_t;
*t = p2m_invalid;
if ( P2M_ROOT_PAGES > 1 )
{
/*
* Concatenated root-level tables. The table number will be
* the offset at the previous level. It is not possible to
* concatenate a level-0 root.
*/
ASSERT(P2M_ROOT_LEVEL > 0);
root_table = offsets[P2M_ROOT_LEVEL - 1];
if ( root_table >= P2M_ROOT_PAGES )
goto err;
}
else
root_table = 0;
map = __map_domain_page(p2m->root + root_table);
ASSERT(P2M_ROOT_LEVEL < 4);
for ( level = P2M_ROOT_LEVEL ; level < 4 ; level++ )
{
mask = masks[level];
pte = map[offsets[level]];
if ( level == 3 && !p2m_table(pte) )
/* Invalid, clobber the pte */
pte.bits = 0;
if ( level == 3 || !p2m_table(pte) )
/* Done */
break;
ASSERT(level < 3);
/* Map for next level */
unmap_domain_page(map);
map = map_domain_page(_mfn(pte.p2m.base));
}
unmap_domain_page(map);
if ( p2m_valid(pte) )
{
ASSERT(mask);
ASSERT(pte.p2m.type != p2m_invalid);
maddr = (pte.bits & PADDR_MASK & mask) | (paddr & ~mask);
*t = pte.p2m.type;
}
err:
return maddr;
}
/* Returns: 0 for success, -errno for failure */
static int
p2m_pt_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
{
/* XXX -- this might be able to be faster iff current->domain == d */
mfn_t table_mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
void *table = map_domain_page(mfn_x(table_mfn));
unsigned long i, gfn_remainder = gfn;
l1_pgentry_t *p2m_entry;
l1_pgentry_t entry_content;
l2_pgentry_t l2e_content;
l3_pgentry_t l3e_content;
int rc;
unsigned int iommu_pte_flags = (p2mt == p2m_ram_rw) ?
IOMMUF_readable|IOMMUF_writable:
0;
unsigned long old_mfn = 0;
if ( tb_init_done )
{
struct {
u64 gfn, mfn;
int p2mt;
int d:16,order:16;
} t;
t.gfn = gfn;
t.mfn = mfn_x(mfn);
t.p2mt = p2mt;
t.d = p2m->domain->domain_id;
t.order = page_order;
__trace_var(TRC_MEM_SET_P2M_ENTRY, 0, sizeof(t), &t);
}
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L4_PAGETABLE_SHIFT - PAGE_SHIFT,
L4_PAGETABLE_ENTRIES, PGT_l3_page_table);
if ( rc )
goto out;
/*
* Try to allocate 1GB page table if this feature is supported.
*/
if ( page_order == PAGE_ORDER_1G )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
l3e_content = mfn_valid(mfn)
? l3e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) | _PAGE_PSE)
: l3e_empty();
entry_content.l1 = l3e_content.l3;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 3);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
else
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder,
gfn, L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES, PGT_l2_page_table);
if ( rc )
goto out;
}
if ( page_order == PAGE_ORDER_4K )
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES, PGT_l1_page_table);
if ( rc )
goto out;
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
0, L1_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( mfn_valid(mfn) || (p2mt == p2m_mmio_direct)
|| p2m_is_paging(p2mt) )
entry_content = p2m_l1e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn));
else
entry_content = l1e_empty();
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
/* level 1 entry */
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 1);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
}
else if ( page_order == PAGE_ORDER_2M )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
/* FIXME: Deal with 4k replaced by 2meg pages */
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
if ( mfn_valid(mfn) || p2m_is_pod(p2mt) )
l2e_content = l2e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) |
_PAGE_PSE);
else
l2e_content = l2e_empty();
entry_content.l1 = l2e_content.l2;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 2);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/* Track the highest gfn for which we have ever had a valid mapping */
if ( p2mt != p2m_invalid
&& (gfn + (1UL << page_order) - 1 > p2m->max_mapped_pfn) )
p2m->max_mapped_pfn = gfn + (1UL << page_order) - 1;
if ( iommu_enabled && need_iommu(p2m->domain) )
{
if ( iommu_hap_pt_share )
{
if ( old_mfn && (old_mfn != mfn_x(mfn)) )
amd_iommu_flush_pages(p2m->domain, gfn, page_order);
}
else
{
if ( p2mt == p2m_ram_rw )
for ( i = 0; i < (1UL << page_order); i++ )
iommu_map_page(p2m->domain, gfn+i, mfn_x(mfn)+i,
IOMMUF_readable|IOMMUF_writable);
else
for ( int i = 0; i < (1UL << page_order); i++ )
iommu_unmap_page(p2m->domain, gfn+i);
}
}
out:
unmap_domain_page(table);
return rc;
}
static int create_p2m_entries(struct domain *d,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t maddr,
int mattr)
{
int rc, flush;
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t *first = NULL, *second = NULL, *third = NULL;
paddr_t addr;
unsigned long cur_first_offset = ~0, cur_second_offset = ~0;
spin_lock(&p2m->lock);
/* XXX Don't actually handle 40 bit guest physical addresses */
BUG_ON(start_gpaddr & 0x8000000000ULL);
BUG_ON(end_gpaddr & 0x8000000000ULL);
first = __map_domain_page(p2m->first_level);
for(addr = start_gpaddr; addr < end_gpaddr; addr += PAGE_SIZE)
{
if ( !first[first_table_offset(addr)].p2m.valid )
{
rc = p2m_create_table(d, &first[first_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L1 failed\n");
goto out;
}
}
BUG_ON(!first[first_table_offset(addr)].p2m.valid);
if ( cur_first_offset != first_table_offset(addr) )
{
if (second) unmap_domain_page(second);
second = map_domain_page(first[first_table_offset(addr)].p2m.base);
cur_first_offset = first_table_offset(addr);
}
/* else: second already valid */
if ( !second[second_table_offset(addr)].p2m.valid )
{
rc = p2m_create_table(d, &second[second_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L2 failed\n");
goto out;
}
}
BUG_ON(!second[second_table_offset(addr)].p2m.valid);
if ( cur_second_offset != second_table_offset(addr) )
{
/* map third level */
if (third) unmap_domain_page(third);
third = map_domain_page(second[second_table_offset(addr)].p2m.base);
cur_second_offset = second_table_offset(addr);
}
flush = third[third_table_offset(addr)].p2m.valid;
/* Allocate a new RAM page and attach */
switch (op) {
case ALLOCATE:
{
struct page_info *page;
lpae_t pte;
rc = -ENOMEM;
page = alloc_domheap_page(d, 0);
if ( page == NULL ) {
printk("p2m_populate_ram: failed to allocate page\n");
goto out;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr);
write_pte(&third[third_table_offset(addr)], pte);
}
break;
case INSERT:
{
lpae_t pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT, mattr);
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
case REMOVE:
{
lpae_t pte;
memset(&pte, 0x00, sizeof(pte));
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
}
if ( flush )
flush_tlb_all_local();
}
rc = 0;
out:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
spin_unlock(&p2m->lock);
return rc;
}
unsigned long hap_p2m_ga_to_gfn(GUEST_PAGING_LEVELS)(
struct vcpu *v, struct p2m_domain *p2m, unsigned long cr3,
paddr_t ga, uint32_t *pfec, unsigned int *page_order)
{
uint32_t missing;
mfn_t top_mfn;
void *top_map;
p2m_type_t p2mt;
walk_t gw;
unsigned long top_gfn;
struct page_info *top_page;
/* Get the top-level table's MFN */
top_gfn = cr3 >> PAGE_SHIFT;
top_page = get_page_from_gfn_p2m(p2m->domain, p2m, top_gfn,
&p2mt, NULL, P2M_ALLOC | P2M_UNSHARE);
if ( p2m_is_paging(p2mt) )
{
ASSERT(p2m_is_hostp2m(p2m));
pfec[0] = PFEC_page_paged;
if ( top_page )
put_page(top_page);
p2m_mem_paging_populate(p2m->domain, cr3 >> PAGE_SHIFT);
return gfn_x(INVALID_GFN);
}
if ( p2m_is_shared(p2mt) )
{
pfec[0] = PFEC_page_shared;
if ( top_page )
put_page(top_page);
return gfn_x(INVALID_GFN);
}
if ( !top_page )
{
pfec[0] &= ~PFEC_page_present;
goto out_tweak_pfec;
}
top_mfn = _mfn(page_to_mfn(top_page));
/* Map the top-level table and call the tree-walker */
ASSERT(mfn_valid(top_mfn));
top_map = map_domain_page(top_mfn);
#if GUEST_PAGING_LEVELS == 3
top_map += (cr3 & ~(PAGE_MASK | 31));
#endif
missing = guest_walk_tables(v, p2m, ga, &gw, pfec[0], top_mfn, top_map);
unmap_domain_page(top_map);
put_page(top_page);
/* Interpret the answer */
if ( missing == 0 )
{
gfn_t gfn = guest_walk_to_gfn(&gw);
struct page_info *page;
page = get_page_from_gfn_p2m(p2m->domain, p2m, gfn_x(gfn), &p2mt,
NULL, P2M_ALLOC | P2M_UNSHARE);
if ( page )
put_page(page);
if ( p2m_is_paging(p2mt) )
{
ASSERT(p2m_is_hostp2m(p2m));
pfec[0] = PFEC_page_paged;
p2m_mem_paging_populate(p2m->domain, gfn_x(gfn));
return gfn_x(INVALID_GFN);
}
if ( p2m_is_shared(p2mt) )
{
pfec[0] = PFEC_page_shared;
return gfn_x(INVALID_GFN);
}
if ( page_order )
*page_order = guest_walk_to_page_order(&gw);
return gfn_x(gfn);
}
if ( missing & _PAGE_PRESENT )
pfec[0] &= ~PFEC_page_present;
if ( missing & _PAGE_INVALID_BITS )
pfec[0] |= PFEC_reserved_bit;
if ( missing & _PAGE_PKEY_BITS )
pfec[0] |= PFEC_prot_key;
if ( missing & _PAGE_PAGED )
pfec[0] = PFEC_page_paged;
if ( missing & _PAGE_SHARED )
pfec[0] = PFEC_page_shared;
out_tweak_pfec:
/*
* SDM Intel 64 Volume 3, Chapter Paging, PAGE-FAULT EXCEPTIONS:
* The PFEC_insn_fetch flag is set only when NX or SMEP are enabled.
*/
if ( !hvm_nx_enabled(v) && !hvm_smep_enabled(v) )
pfec[0] &= ~PFEC_insn_fetch;
return gfn_x(INVALID_GFN);
}
static lpae_t mfn_to_p2m_entry(unsigned long mfn, unsigned int mattr,
p2m_type_t t)
{
paddr_t pa = ((paddr_t) mfn) << PAGE_SHIFT;
/* xn and write bit will be defined in the switch */
lpae_t e = (lpae_t) {
.p2m.af = 1,
.p2m.sh = LPAE_SH_OUTER,
.p2m.read = 1,
.p2m.mattr = mattr,
.p2m.table = 1,
.p2m.valid = 1,
.p2m.type = t,
};
BUILD_BUG_ON(p2m_max_real_type > (1 << 4));
switch (t)
{
case p2m_ram_rw:
e.p2m.xn = 0;
e.p2m.write = 1;
break;
case p2m_ram_ro:
e.p2m.xn = 0;
e.p2m.write = 0;
break;
case p2m_map_foreign:
case p2m_grant_map_rw:
case p2m_mmio_direct:
e.p2m.xn = 1;
e.p2m.write = 1;
break;
case p2m_grant_map_ro:
case p2m_invalid:
e.p2m.xn = 1;
e.p2m.write = 0;
break;
case p2m_max_real_type:
BUG();
break;
}
ASSERT(!(pa & ~PAGE_MASK));
ASSERT(!(pa & ~PADDR_MASK));
e.bits |= pa;
return e;
}
/* Allocate a new page table page and hook it in via the given entry */
static int p2m_create_table(struct domain *d,
lpae_t *entry)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
lpae_t pte;
BUG_ON(entry->p2m.valid);
page = alloc_domheap_page(NULL, 0);
if ( page == NULL )
return -ENOMEM;
page_list_add(page, &p2m->pages);
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM, p2m_invalid);
write_pte(entry, pte);
return 0;
}
enum p2m_operation {
INSERT,
ALLOCATE,
REMOVE,
RELINQUISH,
CACHEFLUSH,
};
static int apply_p2m_changes(struct domain *d,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t maddr,
int mattr,
p2m_type_t t)
{
int rc;
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t *first = NULL, *second = NULL, *third = NULL;
paddr_t addr;
unsigned long cur_first_page = ~0,
cur_first_offset = ~0,
cur_second_offset = ~0;
unsigned long count = 0;
unsigned int flush = 0;
bool_t populate = (op == INSERT || op == ALLOCATE);
lpae_t pte;
spin_lock(&p2m->lock);
if ( d != current->domain )
p2m_load_VTTBR(d);
addr = start_gpaddr;
while ( addr < end_gpaddr )
{
if ( cur_first_page != p2m_first_level_index(addr) )
{
if ( first ) unmap_domain_page(first);
first = p2m_map_first(p2m, addr);
if ( !first )
{
rc = -EINVAL;
goto out;
}
cur_first_page = p2m_first_level_index(addr);
}
if ( !first[first_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + FIRST_SIZE) & FIRST_MASK;
continue;
}
rc = p2m_create_table(d, &first[first_table_offset(addr)]);
if ( rc < 0 )
{
printk("p2m_populate_ram: L1 failed\n");
goto out;
}
}
BUG_ON(!first[first_table_offset(addr)].p2m.valid);
if ( cur_first_offset != first_table_offset(addr) )
{
if (second) unmap_domain_page(second);
second = map_domain_page(first[first_table_offset(addr)].p2m.base);
cur_first_offset = first_table_offset(addr);
}
/* else: second already valid */
if ( !second[second_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + SECOND_SIZE) & SECOND_MASK;
continue;
}
rc = p2m_create_table(d, &second[second_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L2 failed\n");
goto out;
}
}
BUG_ON(!second[second_table_offset(addr)].p2m.valid);
if ( cur_second_offset != second_table_offset(addr) )
{
/* map third level */
if (third) unmap_domain_page(third);
third = map_domain_page(second[second_table_offset(addr)].p2m.base);
cur_second_offset = second_table_offset(addr);
}
pte = third[third_table_offset(addr)];
flush |= pte.p2m.valid;
/* TODO: Handle other p2m type
*
* It's safe to do the put_page here because page_alloc will
* flush the TLBs if the page is reallocated before the end of
* this loop.
*/
if ( pte.p2m.valid && p2m_is_foreign(pte.p2m.type) )
{
unsigned long mfn = pte.p2m.base;
ASSERT(mfn_valid(mfn));
put_page(mfn_to_page(mfn));
}
/* Allocate a new RAM page and attach */
switch (op) {
case ALLOCATE:
{
struct page_info *page;
ASSERT(!pte.p2m.valid);
rc = -ENOMEM;
page = alloc_domheap_page(d, 0);
if ( page == NULL ) {
printk("p2m_populate_ram: failed to allocate page\n");
goto out;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
}
break;
case INSERT:
{
pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT, mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
case RELINQUISH:
case REMOVE:
{
if ( !pte.p2m.valid )
{
count++;
break;
}
count += 0x10;
memset(&pte, 0x00, sizeof(pte));
write_pte(&third[third_table_offset(addr)], pte);
count++;
}
break;
case CACHEFLUSH:
{
if ( !pte.p2m.valid || !p2m_is_ram(pte.p2m.type) )
break;
flush_page_to_ram(pte.p2m.base);
}
break;
}
/* Preempt every 2MiB (mapped) or 32 MiB (unmapped) - arbitrary */
if ( op == RELINQUISH && count >= 0x2000 )
{
if ( hypercall_preempt_check() )
{
p2m->lowest_mapped_gfn = addr >> PAGE_SHIFT;
rc = -EAGAIN;
goto out;
}
count = 0;
}
/* Got the next page */
addr += PAGE_SIZE;
}
if ( flush )
{
/* At the beginning of the function, Xen is updating VTTBR
* with the domain where the mappings are created. In this
* case it's only necessary to flush TLBs on every CPUs with
* the current VMID (our domain).
*/
flush_tlb();
}
if ( op == ALLOCATE || op == INSERT )
{
unsigned long sgfn = paddr_to_pfn(start_gpaddr);
unsigned long egfn = paddr_to_pfn(end_gpaddr);
p2m->max_mapped_gfn = MAX(p2m->max_mapped_gfn, egfn);
p2m->lowest_mapped_gfn = MIN(p2m->lowest_mapped_gfn, sgfn);
}
rc = 0;
out:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
if ( d != current->domain )
p2m_load_VTTBR(current->domain);
spin_unlock(&p2m->lock);
return rc;
}
void dump_pt_walk(paddr_t ttbr, paddr_t addr,
unsigned int root_level,
unsigned int nr_root_tables)
{
static const char *level_strs[4] = { "0TH", "1ST", "2ND", "3RD" };
const unsigned long root_pfn = paddr_to_pfn(ttbr);
const unsigned int offsets[4] = {
zeroeth_table_offset(addr),
first_table_offset(addr),
second_table_offset(addr),
third_table_offset(addr)
};
lpae_t pte, *mapping;
unsigned int level, root_table;
#ifdef CONFIG_ARM_32
BUG_ON(root_level < 1);
#endif
BUG_ON(root_level > 3);
if ( nr_root_tables > 1 )
{
/*
* Concatenated root-level tables. The table number will be
* the offset at the previous level. It is not possible to
* concatenate a level-0 root.
*/
BUG_ON(root_level == 0);
root_table = offsets[root_level - 1];
printk("Using concatenated root table %u\n", root_table);
if ( root_table >= nr_root_tables )
{
printk("Invalid root table offset\n");
return;
}
}
else
root_table = 0;
mapping = map_domain_page(_mfn(root_pfn + root_table));
for ( level = root_level; ; level++ )
{
if ( offsets[level] > LPAE_ENTRIES )
break;
pte = mapping[offsets[level]];
printk("%s[0x%x] = 0x%"PRIpaddr"\n",
level_strs[level], offsets[level], pte.bits);
if ( level == 3 || !pte.walk.valid || !pte.walk.table )
break;
/* For next iteration */
unmap_domain_page(mapping);
mapping = map_domain_page(_mfn(pte.walk.base));
}
unmap_domain_page(mapping);
}
unsigned long hap_p2m_ga_to_gfn(GUEST_PAGING_LEVELS)(
struct vcpu *v, struct p2m_domain *p2m, unsigned long cr3,
paddr_t ga, uint32_t *pfec, unsigned int *page_order)
{
uint32_t missing;
mfn_t top_mfn;
void *top_map;
p2m_type_t p2mt;
walk_t gw;
unsigned long top_gfn;
struct page_info *top_page;
/* Get the top-level table's MFN */
top_gfn = cr3 >> PAGE_SHIFT;
top_page = get_page_from_gfn_p2m(p2m->domain, p2m, top_gfn,
&p2mt, NULL, P2M_ALLOC | P2M_UNSHARE);
if ( p2m_is_paging(p2mt) )
{
ASSERT(p2m_is_hostp2m(p2m));
pfec[0] = PFEC_page_paged;
if ( top_page )
put_page(top_page);
p2m_mem_paging_populate(p2m->domain, cr3 >> PAGE_SHIFT);
return INVALID_GFN;
}
if ( p2m_is_shared(p2mt) )
{
pfec[0] = PFEC_page_shared;
if ( top_page )
put_page(top_page);
return INVALID_GFN;
}
if ( !top_page )
{
pfec[0] &= ~PFEC_page_present;
return INVALID_GFN;
}
top_mfn = _mfn(page_to_mfn(top_page));
/* Map the top-level table and call the tree-walker */
ASSERT(mfn_valid(mfn_x(top_mfn)));
top_map = map_domain_page(mfn_x(top_mfn));
#if GUEST_PAGING_LEVELS == 3
top_map += (cr3 & ~(PAGE_MASK | 31));
#endif
missing = guest_walk_tables(v, p2m, ga, &gw, pfec[0], top_mfn, top_map);
unmap_domain_page(top_map);
put_page(top_page);
/* Interpret the answer */
if ( missing == 0 )
{
gfn_t gfn = guest_l1e_get_gfn(gw.l1e);
struct page_info *page;
page = get_page_from_gfn_p2m(p2m->domain, p2m, gfn_x(gfn), &p2mt,
NULL, P2M_ALLOC | P2M_UNSHARE);
if ( page )
put_page(page);
if ( p2m_is_paging(p2mt) )
{
ASSERT(p2m_is_hostp2m(p2m));
pfec[0] = PFEC_page_paged;
p2m_mem_paging_populate(p2m->domain, gfn_x(gfn));
return INVALID_GFN;
}
if ( p2m_is_shared(p2mt) )
{
pfec[0] = PFEC_page_shared;
return INVALID_GFN;
}
if ( page_order )
*page_order = guest_walk_to_page_order(&gw);
return gfn_x(gfn);
}
if ( missing & _PAGE_PRESENT )
pfec[0] &= ~PFEC_page_present;
if ( missing & _PAGE_INVALID_BITS )
pfec[0] |= PFEC_reserved_bit;
if ( missing & _PAGE_PAGED )
pfec[0] = PFEC_page_paged;
if ( missing & _PAGE_SHARED )
pfec[0] = PFEC_page_shared;
return INVALID_GFN;
}
/*
* pgd3val: this is the value of init_mm.pgd[3] in a PV guest. It is optional.
* This to assist debug of modules in the guest. The kernel address
* space seems is always mapped, but modules are not necessarily
* mapped in any arbitraty guest cr3 that we pick if pgd3val is 0.
* Modules should always be addressible if we use cr3 from init_mm.
* Since pgd3val is already a pgd value, cr3->pgd[3], we just need to
* do 2 level lookups.
*
* NOTE: 4 level paging works for 32 PAE guests also because cpu runs in IA32-e
* mode.
* Returns: mfn for the given (pv guest) vaddr
*/
static mfn_t
dbg_pv_va2mfn(dbgva_t vaddr, struct domain *dp, uint64_t pgd3val)
{
l4_pgentry_t l4e, *l4t;
l3_pgentry_t l3e, *l3t;
l2_pgentry_t l2e, *l2t;
l1_pgentry_t l1e, *l1t;
unsigned long cr3 = (pgd3val ? pgd3val : dp->vcpu[0]->arch.cr3);
mfn_t mfn = _mfn(cr3 >> PAGE_SHIFT);
DBGP2("vaddr:%lx domid:%d cr3:%lx pgd3:%lx\n", vaddr, dp->domain_id,
cr3, pgd3val);
if ( pgd3val == 0 )
{
l4t = map_domain_page(mfn);
l4e = l4t[l4_table_offset(vaddr)];
unmap_domain_page(l4t);
mfn = _mfn(l4e_get_pfn(l4e));
DBGP2("l4t:%p l4to:%lx l4e:%lx mfn:%#"PRI_mfn"\n", l4t,
l4_table_offset(vaddr), l4e, mfn_x(mfn));
if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) )
{
DBGP1("l4 PAGE not present. vaddr:%lx cr3:%lx\n", vaddr, cr3);
return INVALID_MFN;
}
l3t = map_domain_page(mfn);
l3e = l3t[l3_table_offset(vaddr)];
unmap_domain_page(l3t);
mfn = _mfn(l3e_get_pfn(l3e));
DBGP2("l3t:%p l3to:%lx l3e:%lx mfn:%#"PRI_mfn"\n", l3t,
l3_table_offset(vaddr), l3e, mfn_x(mfn));
if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) ||
(l3e_get_flags(l3e) & _PAGE_PSE) )
{
DBGP1("l3 PAGE not present. vaddr:%lx cr3:%lx\n", vaddr, cr3);
return INVALID_MFN;
}
}
l2t = map_domain_page(mfn);
l2e = l2t[l2_table_offset(vaddr)];
unmap_domain_page(l2t);
mfn = _mfn(l2e_get_pfn(l2e));
DBGP2("l2t:%p l2to:%lx l2e:%lx mfn:%#"PRI_mfn"\n",
l2t, l2_table_offset(vaddr), l2e, mfn_x(mfn));
if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) ||
(l2e_get_flags(l2e) & _PAGE_PSE) )
{
DBGP1("l2 PAGE not present. vaddr:%lx cr3:%lx\n", vaddr, cr3);
return INVALID_MFN;
}
l1t = map_domain_page(mfn);
l1e = l1t[l1_table_offset(vaddr)];
unmap_domain_page(l1t);
mfn = _mfn(l1e_get_pfn(l1e));
DBGP2("l1t:%p l1to:%lx l1e:%lx mfn:%#"PRI_mfn"\n", l1t, l1_table_offset(vaddr),
l1e, mfn_x(mfn));
return mfn_valid(mfn_x(mfn)) ? mfn : INVALID_MFN;
}
