/*
* Handle possibly necessary P2M type re-calculation (U flag clear for a
* present entry) for the entries in the page table hierarchy for the given
* GFN. Propagate the re-calculation flag down to the next page table level
* for entries not involved in the translation of the given GFN.
*/
static int do_recalc(struct p2m_domain *p2m, unsigned long gfn)
{
void *table;
unsigned long gfn_remainder = gfn;
unsigned int level = 4;
l1_pgentry_t *pent;
int err = 0;
table = map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
while ( --level )
{
unsigned long remainder = gfn_remainder;
pent = p2m_find_entry(table, &remainder, gfn,
level * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER);
if ( !pent || !(l1e_get_flags(*pent) & _PAGE_PRESENT) )
goto out;
if ( l1e_get_flags(*pent) & _PAGE_PSE )
{
unsigned long mask = ~0UL << (level * PAGETABLE_ORDER);
if ( !needs_recalc(l1, *pent) ||
!p2m_is_changeable(p2m_flags_to_type(l1e_get_flags(*pent))) ||
p2m_is_logdirty_range(p2m, gfn & mask, gfn | ~mask) >= 0 )
break;
}
err = p2m_next_level(p2m, &table, &gfn_remainder, gfn,
level * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER,
pgt[level - 1], 0);
if ( err )
goto out;
if ( needs_recalc(l1, *pent) )
{
l1_pgentry_t e = *pent, *ptab = table;
unsigned int i;
if ( !valid_recalc(l1, e) )
P2M_DEBUG("bogus recalc state at d%d:%lx:%u\n",
p2m->domain->domain_id, gfn, level);
remainder = gfn_remainder;
for ( i = 0; i < (1 << PAGETABLE_ORDER); ++i )
{
l1_pgentry_t ent = ptab[i];
if ( (l1e_get_flags(ent) & _PAGE_PRESENT) &&
!needs_recalc(l1, ent) )
{
set_recalc(l1, ent);
p2m->write_p2m_entry(p2m, gfn - remainder, &ptab[i],
ent, level);
}
remainder -= 1UL << ((level - 1) * PAGETABLE_ORDER);
}
smp_wmb();
clear_recalc(l1, e);
p2m->write_p2m_entry(p2m, gfn, pent, e, level + 1);
}
unmap_domain_page((void *)((unsigned long)pent & PAGE_MASK));
}
pent = p2m_find_entry(table, &gfn_remainder, gfn,
level * PAGETABLE_ORDER, 1 << PAGETABLE_ORDER);
if ( pent && (l1e_get_flags(*pent) & _PAGE_PRESENT) &&
needs_recalc(l1, *pent) )
{
l1_pgentry_t e = *pent;
if ( !valid_recalc(l1, e) )
P2M_DEBUG("bogus recalc leaf at d%d:%lx:%u\n",
p2m->domain->domain_id, gfn, level);
if ( p2m_is_changeable(p2m_flags_to_type(l1e_get_flags(e))) )
{
unsigned long mask = ~0UL << (level * PAGETABLE_ORDER);
p2m_type_t p2mt = p2m_is_logdirty_range(p2m, gfn & mask, gfn | ~mask)
? p2m_ram_logdirty : p2m_ram_rw;
unsigned long mfn = l1e_get_pfn(e);
unsigned long flags = p2m_type_to_flags(p2mt, _mfn(mfn));
if ( level )
{
if ( flags & _PAGE_PAT )
{
BUILD_BUG_ON(_PAGE_PAT != _PAGE_PSE);
mfn |= _PAGE_PSE_PAT >> PAGE_SHIFT;
}
else
mfn &= ~(_PAGE_PSE_PAT >> PAGE_SHIFT);
flags |= _PAGE_PSE;
}
long p2m_pt_audit_p2m(struct p2m_domain *p2m)
{
unsigned long entry_count = 0, pmbad = 0;
unsigned long mfn, gfn, m2pfn;
int test_linear;
struct domain *d = p2m->domain;
ASSERT(p2m_locked_by_me(p2m));
ASSERT(pod_locked_by_me(p2m));
test_linear = ( (d == current->domain)
&& !pagetable_is_null(current->arch.monitor_table) );
if ( test_linear )
flush_tlb_local();
/* Audit part one: walk the domain's p2m table, checking the entries. */
if ( pagetable_get_pfn(p2m_get_pagetable(p2m)) != 0 )
{
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
int i1, i2;
#if CONFIG_PAGING_LEVELS == 4
l4_pgentry_t *l4e;
l3_pgentry_t *l3e;
int i4, i3;
l4e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#else /* CONFIG_PAGING_LEVELS == 3 */
l3_pgentry_t *l3e;
int i3;
l3e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#endif
gfn = 0;
#if CONFIG_PAGING_LEVELS >= 4
for ( i4 = 0; i4 < L4_PAGETABLE_ENTRIES; i4++ )
{
if ( !(l4e_get_flags(l4e[i4]) & _PAGE_PRESENT) )
{
gfn += 1 << (L4_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
l3e = map_domain_page(mfn_x(_mfn(l4e_get_pfn(l4e[i4]))));
#endif
for ( i3 = 0;
i3 < ((CONFIG_PAGING_LEVELS==4) ? L3_PAGETABLE_ENTRIES : 8);
i3++ )
{
if ( !(l3e_get_flags(l3e[i3]) & _PAGE_PRESENT) )
{
gfn += 1 << (L3_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
/* check for 1GB super page */
if ( l3e_get_flags(l3e[i3]) & _PAGE_PSE )
{
mfn = l3e_get_pfn(l3e[i3]);
ASSERT(mfn_valid(_mfn(mfn)));
/* we have to cover 512x512 4K pages */
for ( i2 = 0;
i2 < (L2_PAGETABLE_ENTRIES * L1_PAGETABLE_ENTRIES);
i2++)
{
m2pfn = get_gpfn_from_mfn(mfn+i2);
if ( m2pfn != (gfn + i2) )
{
pmbad++;
P2M_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn+i2, mfn+i2,
m2pfn);
BUG();
}
gfn += 1 << (L3_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
}
l2e = map_domain_page(mfn_x(_mfn(l3e_get_pfn(l3e[i3]))));
for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; i2++ )
{
if ( !(l2e_get_flags(l2e[i2]) & _PAGE_PRESENT) )
{
if ( (l2e_get_flags(l2e[i2]) & _PAGE_PSE)
&& ( p2m_flags_to_type(l2e_get_flags(l2e[i2]))
== p2m_populate_on_demand ) )
entry_count+=SUPERPAGE_PAGES;
gfn += 1 << (L2_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
/* check for super page */
if ( l2e_get_flags(l2e[i2]) & _PAGE_PSE )
{
mfn = l2e_get_pfn(l2e[i2]);
ASSERT(mfn_valid(_mfn(mfn)));
for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++)
{
m2pfn = get_gpfn_from_mfn(mfn+i1);
/* Allow shared M2Ps */
if ( (m2pfn != (gfn + i1)) &&
(m2pfn != SHARED_M2P_ENTRY) )
{
pmbad++;
P2M_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn+i1, mfn+i1,
m2pfn);
BUG();
}
}
gfn += 1 << (L2_PAGETABLE_SHIFT - PAGE_SHIFT);
continue;
}
l1e = map_domain_page(mfn_x(_mfn(l2e_get_pfn(l2e[i2]))));
for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++, gfn++ )
{
p2m_type_t type;
type = p2m_flags_to_type(l1e_get_flags(l1e[i1]));
if ( !(l1e_get_flags(l1e[i1]) & _PAGE_PRESENT) )
{
if ( type == p2m_populate_on_demand )
entry_count++;
continue;
}
mfn = l1e_get_pfn(l1e[i1]);
ASSERT(mfn_valid(_mfn(mfn)));
m2pfn = get_gpfn_from_mfn(mfn);
if ( m2pfn != gfn &&
type != p2m_mmio_direct &&
!p2m_is_grant(type) &&
!p2m_is_shared(type) )
{
pmbad++;
printk("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn, mfn, m2pfn);
P2M_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
" -> gfn %#lx\n", gfn, mfn, m2pfn);
BUG();
}
}
unmap_domain_page(l1e);
}
unmap_domain_page(l2e);
}
#if CONFIG_PAGING_LEVELS >= 4
unmap_domain_page(l3e);
}
#endif
#if CONFIG_PAGING_LEVELS == 4
unmap_domain_page(l4e);
#else /* CONFIG_PAGING_LEVELS == 3 */
unmap_domain_page(l3e);
#endif
}
if ( entry_count != p2m->pod.entry_count )
{
printk("%s: refcounted entry count %ld, audit count %lu!\n",
__func__,
p2m->pod.entry_count,
entry_count);
BUG();
}
return pmbad;
}
/* Walk the whole p2m table, changing any entries of the old type
* to the new type. This is used in hardware-assisted paging to
* quickly enable or diable log-dirty tracking */
static void p2m_change_type_global(struct p2m_domain *p2m,
p2m_type_t ot, p2m_type_t nt)
{
unsigned long mfn, gfn, flags;
l1_pgentry_t l1e_content;
l1_pgentry_t *l1e;
l2_pgentry_t *l2e;
mfn_t l1mfn, l2mfn, l3mfn;
unsigned long i1, i2, i3;
l3_pgentry_t *l3e;
#if CONFIG_PAGING_LEVELS == 4
l4_pgentry_t *l4e;
unsigned long i4;
#endif /* CONFIG_PAGING_LEVELS == 4 */
BUG_ON(p2m_is_grant(ot) || p2m_is_grant(nt));
BUG_ON(ot != nt && (ot == p2m_mmio_direct || nt == p2m_mmio_direct));
if ( !paging_mode_translate(p2m->domain) )
return;
if ( pagetable_get_pfn(p2m_get_pagetable(p2m)) == 0 )
return;
ASSERT(p2m_locked_by_me(p2m));
#if CONFIG_PAGING_LEVELS == 4
l4e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#else /* CONFIG_PAGING_LEVELS == 3 */
l3mfn = _mfn(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
l3e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#endif
#if CONFIG_PAGING_LEVELS >= 4
for ( i4 = 0; i4 < L4_PAGETABLE_ENTRIES; i4++ )
{
if ( !(l4e_get_flags(l4e[i4]) & _PAGE_PRESENT) )
{
continue;
}
l3mfn = _mfn(l4e_get_pfn(l4e[i4]));
l3e = map_domain_page(l4e_get_pfn(l4e[i4]));
#endif
for ( i3 = 0;
i3 < ((CONFIG_PAGING_LEVELS==4) ? L3_PAGETABLE_ENTRIES : 8);
i3++ )
{
if ( !(l3e_get_flags(l3e[i3]) & _PAGE_PRESENT) )
{
continue;
}
if ( (l3e_get_flags(l3e[i3]) & _PAGE_PSE) )
{
flags = l3e_get_flags(l3e[i3]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l3e_get_pfn(l3e[i3]);
gfn = get_gpfn_from_mfn(mfn);
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = l1e_from_pfn(mfn, flags | _PAGE_PSE);
p2m->write_p2m_entry(p2m, gfn,
(l1_pgentry_t *)&l3e[i3],
l3mfn, l1e_content, 3);
continue;
}
l2mfn = _mfn(l3e_get_pfn(l3e[i3]));
l2e = map_domain_page(l3e_get_pfn(l3e[i3]));
for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; i2++ )
{
if ( !(l2e_get_flags(l2e[i2]) & _PAGE_PRESENT) )
{
continue;
}
if ( (l2e_get_flags(l2e[i2]) & _PAGE_PSE) )
{
flags = l2e_get_flags(l2e[i2]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l2e_get_pfn(l2e[i2]);
/* Do not use get_gpfn_from_mfn because it may return
SHARED_M2P_ENTRY */
gfn = (i2 + (i3
#if CONFIG_PAGING_LEVELS >= 4
+ (i4 * L3_PAGETABLE_ENTRIES)
#endif
)
* L2_PAGETABLE_ENTRIES) * L1_PAGETABLE_ENTRIES;
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = l1e_from_pfn(mfn, flags | _PAGE_PSE);
p2m->write_p2m_entry(p2m, gfn,
(l1_pgentry_t *)&l2e[i2],
l2mfn, l1e_content, 2);
continue;
}
l1mfn = _mfn(l2e_get_pfn(l2e[i2]));
l1e = map_domain_page(mfn_x(l1mfn));
for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++, gfn++ )
{
flags = l1e_get_flags(l1e[i1]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l1e_get_pfn(l1e[i1]);
gfn = i1 + (i2 + (i3
#if CONFIG_PAGING_LEVELS >= 4
+ (i4 * L3_PAGETABLE_ENTRIES)
#endif
)
* L2_PAGETABLE_ENTRIES) * L1_PAGETABLE_ENTRIES;
/* create a new 1le entry with the new type */
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = p2m_l1e_from_pfn(mfn, flags);
p2m->write_p2m_entry(p2m, gfn, &l1e[i1],
l1mfn, l1e_content, 1);
}
unmap_domain_page(l1e);
}
unmap_domain_page(l2e);
}
#if CONFIG_PAGING_LEVELS >= 4
unmap_domain_page(l3e);
}
#endif
#if CONFIG_PAGING_LEVELS == 4
unmap_domain_page(l4e);
#else /* CONFIG_PAGING_LEVELS == 3 */
unmap_domain_page(l3e);
#endif
}
static mfn_t
p2m_gfn_to_mfn(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
unsigned long l1e_flags;
p2m_type_t l1t;
ASSERT(paging_mode_translate(p2m->domain));
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
*t = p2m_mmio_dm;
/* Not implemented except with EPT */
*a = p2m_access_rwx;
if ( gfn > p2m->max_mapped_pfn )
/* This pfn is higher than the highest the p2m map currently holds */
return _mfn(INVALID_MFN);
/* Use the fast path with the linear mapping if we can */
if ( p2m == p2m_get_hostp2m(current->domain) )
return p2m_gfn_to_mfn_current(p2m, gfn, t, a, q, page_order);
mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
#if CONFIG_PAGING_LEVELS >= 4
{
l4_pgentry_t *l4e = map_domain_page(mfn_x(mfn));
l4e += l4_table_offset(addr);
if ( (l4e_get_flags(*l4e) & _PAGE_PRESENT) == 0 )
{
unmap_domain_page(l4e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l4e_get_pfn(*l4e));
unmap_domain_page(l4e);
}
#endif
{
l3_pgentry_t *l3e = map_domain_page(mfn_x(mfn));
#if CONFIG_PAGING_LEVELS == 3
/* On PAE hosts the p2m has eight l3 entries, not four (see
* shadow_set_p2m_entry()) so we can't use l3_table_offset.
* Instead, just count the number of l3es from zero. It's safe
* to do this because we already checked that the gfn is within
* the bounds of the p2m. */
l3e += (addr >> L3_PAGETABLE_SHIFT);
#else
l3e += l3_table_offset(addr);
#endif
pod_retry_l3:
if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
{
if ( p2m_flags_to_type(l3e_get_flags(*l3e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
}
else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l3e);
return _mfn(INVALID_MFN);
}
else if ( (l3e_get_flags(*l3e) & _PAGE_PSE) )
{
mfn = _mfn(l3e_get_pfn(*l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
*t = p2m_flags_to_type(l3e_get_flags(*l3e));
unmap_domain_page(l3e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_1G;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l3e_get_pfn(*l3e));
unmap_domain_page(l3e);
}
l2e = map_domain_page(mfn_x(mfn));
l2e += l2_table_offset(addr);
pod_retry_l2:
if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )
{
/* PoD: Try to populate a 2-meg chunk */
if ( p2m_flags_to_type(l2e_get_flags(*l2e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_2M, q) )
goto pod_retry_l2;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l2e);
return _mfn(INVALID_MFN);
}
else if ( (l2e_get_flags(*l2e) & _PAGE_PSE) )
{
mfn = _mfn(l2e_get_pfn(*l2e) + l1_table_offset(addr));
*t = p2m_flags_to_type(l2e_get_flags(*l2e));
unmap_domain_page(l2e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_2M;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l2e_get_pfn(*l2e));
unmap_domain_page(l2e);
l1e = map_domain_page(mfn_x(mfn));
l1e += l1_table_offset(addr);
pod_retry_l1:
l1e_flags = l1e_get_flags(*l1e);
l1t = p2m_flags_to_type(l1e_flags);
if ( ((l1e_flags & _PAGE_PRESENT) == 0) && (!p2m_is_paging(l1t)) )
{
/* PoD: Try to populate */
if ( l1t == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_4K, q) )
goto pod_retry_l1;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l1e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l1e_get_pfn(*l1e));
*t = l1t;
unmap_domain_page(l1e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t) || p2m_is_paging(*t));
if ( page_order )
*page_order = PAGE_ORDER_4K;
return (p2m_is_valid(*t) || p2m_is_grant(*t)) ? mfn : _mfn(INVALID_MFN);
}
/* Read the current domain's p2m table (through the linear mapping). */
static mfn_t p2m_gfn_to_mfn_current(struct p2m_domain *p2m,
unsigned long gfn, p2m_type_t *t,
p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn = _mfn(INVALID_MFN);
p2m_type_t p2mt = p2m_mmio_dm;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
l1_pgentry_t l1e = l1e_empty(), *p2m_entry;
l2_pgentry_t l2e = l2e_empty();
int ret;
#if CONFIG_PAGING_LEVELS >= 4
l3_pgentry_t l3e = l3e_empty();
#endif
ASSERT(gfn < (RO_MPT_VIRT_END - RO_MPT_VIRT_START)
/ sizeof(l1_pgentry_t));
#if CONFIG_PAGING_LEVELS >= 4
/*
* Read & process L3
*/
p2m_entry = (l1_pgentry_t *)
&__linear_l2_table[l2_linear_offset(RO_MPT_VIRT_START)
+ l3_linear_offset(addr)];
pod_retry_l3:
ret = __copy_from_user(&l3e, p2m_entry, sizeof(l3e));
if ( ret != 0 || !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
{
if ( (l3e_get_flags(l3e) & _PAGE_PSE) &&
(p2m_flags_to_type(l3e_get_flags(l3e)) == p2m_populate_on_demand) )
{
/* The read has succeeded, so we know that mapping exists */
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
p2mt = p2m_invalid;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
goto out;
}
else
{
p2mt = p2m_populate_on_demand;
goto out;
}
}
goto pod_retry_l2;
}
if ( l3e_get_flags(l3e) & _PAGE_PSE )
{
p2mt = p2m_flags_to_type(l3e_get_flags(l3e));
ASSERT(l3e_get_pfn(l3e) != INVALID_MFN || !p2m_is_ram(p2mt));
if (p2m_is_valid(p2mt) )
mfn = _mfn(l3e_get_pfn(l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
else
p2mt = p2m_mmio_dm;
if ( page_order )
*page_order = PAGE_ORDER_1G;
goto out;
}
#endif
/*
* Read & process L2
*/
p2m_entry = &__linear_l1_table[l1_linear_offset(RO_MPT_VIRT_START)
+ l2_linear_offset(addr)];
pod_retry_l2:
ret = __copy_from_user(&l2e,
p2m_entry,
sizeof(l2e));
if ( ret != 0
|| !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
{
if( (l2e_get_flags(l2e) & _PAGE_PSE)
&& ( p2m_flags_to_type(l2e_get_flags(l2e))
== p2m_populate_on_demand ) )
{
/* The read has succeeded, so we know that the mapping
* exits at this point. */
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn,
PAGE_ORDER_2M, q) )
goto pod_retry_l2;
/* Allocate failed. */
p2mt = p2m_invalid;
printk("%s: Allocate failed!\n", __func__);
goto out;
}
else
{
p2mt = p2m_populate_on_demand;
goto out;
}
}
goto pod_retry_l1;
}
if (l2e_get_flags(l2e) & _PAGE_PSE)
{
p2mt = p2m_flags_to_type(l2e_get_flags(l2e));
ASSERT(l2e_get_pfn(l2e) != INVALID_MFN || !p2m_is_ram(p2mt));
if ( p2m_is_valid(p2mt) )
mfn = _mfn(l2e_get_pfn(l2e) + l1_table_offset(addr));
else
p2mt = p2m_mmio_dm;
if ( page_order )
*page_order = PAGE_ORDER_2M;
goto out;
}
/*
* Read and process L1
*/
/* Need to __copy_from_user because the p2m is sparse and this
* part might not exist */
pod_retry_l1:
p2m_entry = &phys_to_machine_mapping[gfn];
ret = __copy_from_user(&l1e,
p2m_entry,
sizeof(l1e));
if ( ret == 0 ) {
unsigned long l1e_mfn = l1e_get_pfn(l1e);
p2mt = p2m_flags_to_type(l1e_get_flags(l1e));
ASSERT( mfn_valid(_mfn(l1e_mfn)) || !p2m_is_ram(p2mt) ||
p2m_is_paging(p2mt) );
if ( p2mt == p2m_populate_on_demand )
{
/* The read has succeeded, so we know that the mapping
* exits at this point. */
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn,
PAGE_ORDER_4K, q) )
goto pod_retry_l1;
/* Allocate failed. */
p2mt = p2m_invalid;
goto out;
}
else
{
p2mt = p2m_populate_on_demand;
goto out;
}
}
if ( p2m_is_valid(p2mt) || p2m_is_grant(p2mt) )
mfn = _mfn(l1e_mfn);
else
/* XXX see above */
p2mt = p2m_mmio_dm;
}
if ( page_order )
*page_order = PAGE_ORDER_4K;
out:
*t = p2mt;
return mfn;
}
static mfn_t
p2m_pt_get_entry(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
mfn_t mfn;
paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;
l2_pgentry_t *l2e;
l1_pgentry_t *l1e;
unsigned long l1e_flags;
p2m_type_t l1t;
ASSERT(paging_mode_translate(p2m->domain));
/* XXX This is for compatibility with the old model, where anything not
* XXX marked as RAM was considered to be emulated MMIO space.
* XXX Once we start explicitly registering MMIO regions in the p2m
* XXX we will return p2m_invalid for unmapped gfns */
*t = p2m_mmio_dm;
/* Not implemented except with EPT */
*a = p2m_access_rwx;
if ( gfn > p2m->max_mapped_pfn )
/* This pfn is higher than the highest the p2m map currently holds */
return _mfn(INVALID_MFN);
mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
{
l4_pgentry_t *l4e = map_domain_page(mfn_x(mfn));
l4e += l4_table_offset(addr);
if ( (l4e_get_flags(*l4e) & _PAGE_PRESENT) == 0 )
{
unmap_domain_page(l4e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l4e_get_pfn(*l4e));
unmap_domain_page(l4e);
}
{
l3_pgentry_t *l3e = map_domain_page(mfn_x(mfn));
l3e += l3_table_offset(addr);
pod_retry_l3:
if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
{
if ( p2m_flags_to_type(l3e_get_flags(*l3e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC )
{
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_1G, q) )
goto pod_retry_l3;
gdprintk(XENLOG_ERR, "%s: Allocate 1GB failed!\n", __func__);
}
else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l3e);
return _mfn(INVALID_MFN);
}
else if ( (l3e_get_flags(*l3e) & _PAGE_PSE) )
{
mfn = _mfn(l3e_get_pfn(*l3e) +
l2_table_offset(addr) * L1_PAGETABLE_ENTRIES +
l1_table_offset(addr));
*t = p2m_flags_to_type(l3e_get_flags(*l3e));
unmap_domain_page(l3e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_1G;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l3e_get_pfn(*l3e));
unmap_domain_page(l3e);
}
l2e = map_domain_page(mfn_x(mfn));
l2e += l2_table_offset(addr);
pod_retry_l2:
if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )
{
/* PoD: Try to populate a 2-meg chunk */
if ( p2m_flags_to_type(l2e_get_flags(*l2e)) == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_2M, q) )
goto pod_retry_l2;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l2e);
return _mfn(INVALID_MFN);
}
else if ( (l2e_get_flags(*l2e) & _PAGE_PSE) )
{
mfn = _mfn(l2e_get_pfn(*l2e) + l1_table_offset(addr));
*t = p2m_flags_to_type(l2e_get_flags(*l2e));
unmap_domain_page(l2e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t));
if ( page_order )
*page_order = PAGE_ORDER_2M;
return (p2m_is_valid(*t)) ? mfn : _mfn(INVALID_MFN);
}
mfn = _mfn(l2e_get_pfn(*l2e));
unmap_domain_page(l2e);
l1e = map_domain_page(mfn_x(mfn));
l1e += l1_table_offset(addr);
pod_retry_l1:
l1e_flags = l1e_get_flags(*l1e);
l1t = p2m_flags_to_type(l1e_flags);
if ( ((l1e_flags & _PAGE_PRESENT) == 0) && (!p2m_is_paging(l1t)) )
{
/* PoD: Try to populate */
if ( l1t == p2m_populate_on_demand )
{
if ( q & P2M_ALLOC ) {
if ( !p2m_pod_demand_populate(p2m, gfn, PAGE_ORDER_4K, q) )
goto pod_retry_l1;
} else
*t = p2m_populate_on_demand;
}
unmap_domain_page(l1e);
return _mfn(INVALID_MFN);
}
mfn = _mfn(l1e_get_pfn(*l1e));
*t = l1t;
unmap_domain_page(l1e);
ASSERT(mfn_valid(mfn) || !p2m_is_ram(*t) || p2m_is_paging(*t));
if ( page_order )
*page_order = PAGE_ORDER_4K;
return (p2m_is_valid(*t) || p2m_is_grant(*t)) ? mfn : _mfn(INVALID_MFN);
}
