// Returns 0 on error (out of memory)
static int
p2m_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
{
// XXX -- this might be able to be faster iff current->domain == d
mfn_t table_mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
void *table =map_domain_page(mfn_x(table_mfn));
unsigned long i, gfn_remainder = gfn;
l1_pgentry_t *p2m_entry;
l1_pgentry_t entry_content;
l2_pgentry_t l2e_content;
l3_pgentry_t l3e_content;
int rv=0;
unsigned int iommu_pte_flags = (p2mt == p2m_ram_rw) ?
IOMMUF_readable|IOMMUF_writable:
0;
unsigned long old_mfn = 0;
if ( tb_init_done )
{
struct {
u64 gfn, mfn;
int p2mt;
int d:16,order:16;
} t;
t.gfn = gfn;
t.mfn = mfn_x(mfn);
t.p2mt = p2mt;
t.d = p2m->domain->domain_id;
t.order = page_order;
__trace_var(TRC_MEM_SET_P2M_ENTRY, 0, sizeof(t), &t);
}
#if CONFIG_PAGING_LEVELS >= 4
if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L4_PAGETABLE_SHIFT - PAGE_SHIFT,
L4_PAGETABLE_ENTRIES, PGT_l3_page_table) )
goto out;
#endif
/*
* Try to allocate 1GB page table if this feature is supported.
*/
if ( page_order == PAGE_ORDER_1G )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
l3e_content = mfn_valid(mfn)
? l3e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) | _PAGE_PSE)
: l3e_empty();
entry_content.l1 = l3e_content.l3;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 3);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/*
* When using PAE Xen, we only allow 33 bits of pseudo-physical
* address in translated guests (i.e. 8 GBytes). This restriction
* comes from wanting to map the P2M table into the 16MB RO_MPT hole
* in Xen's address space for translated PV guests.
* When using AMD's NPT on PAE Xen, we are restricted to 4GB.
*/
else if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
((CONFIG_PAGING_LEVELS == 3)
? (hap_enabled(p2m->domain) ? 4 : 8)
: L3_PAGETABLE_ENTRIES),
PGT_l2_page_table) )
goto out;
if ( page_order == PAGE_ORDER_4K )
{
if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES, PGT_l1_page_table) )
goto out;
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
0, L1_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( mfn_valid(mfn) || (p2mt == p2m_mmio_direct)
|| p2m_is_paging(p2mt) )
entry_content = p2m_l1e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn));
else
entry_content = l1e_empty();
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
/* level 1 entry */
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 1);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
}
else if ( page_order == PAGE_ORDER_2M )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
/* FIXME: Deal with 4k replaced by 2meg pages */
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
if ( mfn_valid(mfn) || p2m_is_magic(p2mt) )
l2e_content = l2e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) |
_PAGE_PSE);
else
l2e_content = l2e_empty();
entry_content.l1 = l2e_content.l2;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 2);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/* Track the highest gfn for which we have ever had a valid mapping */
if ( p2mt != p2m_invalid
&& (gfn + (1UL << page_order) - 1 > p2m->max_mapped_pfn) )
p2m->max_mapped_pfn = gfn + (1UL << page_order) - 1;
if ( iommu_enabled && need_iommu(p2m->domain) )
{
if ( iommu_hap_pt_share )
{
if ( old_mfn && (old_mfn != mfn_x(mfn)) )
amd_iommu_flush_pages(p2m->domain, gfn, page_order);
}
else
{
if ( p2mt == p2m_ram_rw )
for ( i = 0; i < (1UL << page_order); i++ )
iommu_map_page(p2m->domain, gfn+i, mfn_x(mfn)+i,
IOMMUF_readable|IOMMUF_writable);
else
for ( int i = 0; i < (1UL << page_order); i++ )
iommu_unmap_page(p2m->domain, gfn+i);
}
}
/* Success */
rv = 1;
out:
unmap_domain_page(table);
return rv;
}
/* Walk the whole p2m table, changing any entries of the old type
* to the new type. This is used in hardware-assisted paging to
* quickly enable or diable log-dirty tracking */
static void p2m_change_type_global(struct p2m_domain *p2m,
p2m_type_t ot, p2m_type_t nt)
{
unsigned long mfn, gfn, flags;
l1_pgentry_t l1e_content;
l1_pgentry_t *l1e;
l2_pgentry_t *l2e;
mfn_t l1mfn, l2mfn, l3mfn;
unsigned long i1, i2, i3;
l3_pgentry_t *l3e;
#if CONFIG_PAGING_LEVELS == 4
l4_pgentry_t *l4e;
unsigned long i4;
#endif /* CONFIG_PAGING_LEVELS == 4 */
BUG_ON(p2m_is_grant(ot) || p2m_is_grant(nt));
BUG_ON(ot != nt && (ot == p2m_mmio_direct || nt == p2m_mmio_direct));
if ( !paging_mode_translate(p2m->domain) )
return;
if ( pagetable_get_pfn(p2m_get_pagetable(p2m)) == 0 )
return;
ASSERT(p2m_locked_by_me(p2m));
#if CONFIG_PAGING_LEVELS == 4
l4e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#else /* CONFIG_PAGING_LEVELS == 3 */
l3mfn = _mfn(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
l3e = map_domain_page(mfn_x(pagetable_get_mfn(p2m_get_pagetable(p2m))));
#endif
#if CONFIG_PAGING_LEVELS >= 4
for ( i4 = 0; i4 < L4_PAGETABLE_ENTRIES; i4++ )
{
if ( !(l4e_get_flags(l4e[i4]) & _PAGE_PRESENT) )
{
continue;
}
l3mfn = _mfn(l4e_get_pfn(l4e[i4]));
l3e = map_domain_page(l4e_get_pfn(l4e[i4]));
#endif
for ( i3 = 0;
i3 < ((CONFIG_PAGING_LEVELS==4) ? L3_PAGETABLE_ENTRIES : 8);
i3++ )
{
if ( !(l3e_get_flags(l3e[i3]) & _PAGE_PRESENT) )
{
continue;
}
if ( (l3e_get_flags(l3e[i3]) & _PAGE_PSE) )
{
flags = l3e_get_flags(l3e[i3]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l3e_get_pfn(l3e[i3]);
gfn = get_gpfn_from_mfn(mfn);
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = l1e_from_pfn(mfn, flags | _PAGE_PSE);
p2m->write_p2m_entry(p2m, gfn,
(l1_pgentry_t *)&l3e[i3],
l3mfn, l1e_content, 3);
continue;
}
l2mfn = _mfn(l3e_get_pfn(l3e[i3]));
l2e = map_domain_page(l3e_get_pfn(l3e[i3]));
for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; i2++ )
{
if ( !(l2e_get_flags(l2e[i2]) & _PAGE_PRESENT) )
{
continue;
}
if ( (l2e_get_flags(l2e[i2]) & _PAGE_PSE) )
{
flags = l2e_get_flags(l2e[i2]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l2e_get_pfn(l2e[i2]);
/* Do not use get_gpfn_from_mfn because it may return
SHARED_M2P_ENTRY */
gfn = (i2 + (i3
#if CONFIG_PAGING_LEVELS >= 4
+ (i4 * L3_PAGETABLE_ENTRIES)
#endif
)
* L2_PAGETABLE_ENTRIES) * L1_PAGETABLE_ENTRIES;
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = l1e_from_pfn(mfn, flags | _PAGE_PSE);
p2m->write_p2m_entry(p2m, gfn,
(l1_pgentry_t *)&l2e[i2],
l2mfn, l1e_content, 2);
continue;
}
l1mfn = _mfn(l2e_get_pfn(l2e[i2]));
l1e = map_domain_page(mfn_x(l1mfn));
for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++, gfn++ )
{
flags = l1e_get_flags(l1e[i1]);
if ( p2m_flags_to_type(flags) != ot )
continue;
mfn = l1e_get_pfn(l1e[i1]);
gfn = i1 + (i2 + (i3
#if CONFIG_PAGING_LEVELS >= 4
+ (i4 * L3_PAGETABLE_ENTRIES)
#endif
)
* L2_PAGETABLE_ENTRIES) * L1_PAGETABLE_ENTRIES;
/* create a new 1le entry with the new type */
flags = p2m_type_to_flags(nt, _mfn(mfn));
l1e_content = p2m_l1e_from_pfn(mfn, flags);
p2m->write_p2m_entry(p2m, gfn, &l1e[i1],
l1mfn, l1e_content, 1);
}
unmap_domain_page(l1e);
}
unmap_domain_page(l2e);
}
#if CONFIG_PAGING_LEVELS >= 4
unmap_domain_page(l3e);
}
#endif
#if CONFIG_PAGING_LEVELS == 4
unmap_domain_page(l4e);
#else /* CONFIG_PAGING_LEVELS == 3 */
unmap_domain_page(l3e);
#endif
}
/* Returns: 0 for success, -errno for failure */
static int
p2m_pt_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
{
/* XXX -- this might be able to be faster iff current->domain == d */
mfn_t table_mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
void *table = map_domain_page(mfn_x(table_mfn));
unsigned long i, gfn_remainder = gfn;
l1_pgentry_t *p2m_entry;
l1_pgentry_t entry_content;
l2_pgentry_t l2e_content;
l3_pgentry_t l3e_content;
int rc;
unsigned int iommu_pte_flags = (p2mt == p2m_ram_rw) ?
IOMMUF_readable|IOMMUF_writable:
0;
unsigned long old_mfn = 0;
if ( tb_init_done )
{
struct {
u64 gfn, mfn;
int p2mt;
int d:16,order:16;
} t;
t.gfn = gfn;
t.mfn = mfn_x(mfn);
t.p2mt = p2mt;
t.d = p2m->domain->domain_id;
t.order = page_order;
__trace_var(TRC_MEM_SET_P2M_ENTRY, 0, sizeof(t), &t);
}
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L4_PAGETABLE_SHIFT - PAGE_SHIFT,
L4_PAGETABLE_ENTRIES, PGT_l3_page_table);
if ( rc )
goto out;
/*
* Try to allocate 1GB page table if this feature is supported.
*/
if ( page_order == PAGE_ORDER_1G )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
l3e_content = mfn_valid(mfn)
? l3e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) | _PAGE_PSE)
: l3e_empty();
entry_content.l1 = l3e_content.l3;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 3);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
else
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder,
gfn, L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES, PGT_l2_page_table);
if ( rc )
goto out;
}
if ( page_order == PAGE_ORDER_4K )
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES, PGT_l1_page_table);
if ( rc )
goto out;
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
0, L1_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( mfn_valid(mfn) || (p2mt == p2m_mmio_direct)
|| p2m_is_paging(p2mt) )
entry_content = p2m_l1e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn));
else
entry_content = l1e_empty();
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
/* level 1 entry */
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 1);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
}
else if ( page_order == PAGE_ORDER_2M )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
/* FIXME: Deal with 4k replaced by 2meg pages */
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
if ( mfn_valid(mfn) || p2m_is_pod(p2mt) )
l2e_content = l2e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) |
_PAGE_PSE);
else
l2e_content = l2e_empty();
entry_content.l1 = l2e_content.l2;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 2);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/* Track the highest gfn for which we have ever had a valid mapping */
if ( p2mt != p2m_invalid
&& (gfn + (1UL << page_order) - 1 > p2m->max_mapped_pfn) )
p2m->max_mapped_pfn = gfn + (1UL << page_order) - 1;
if ( iommu_enabled && need_iommu(p2m->domain) )
{
if ( iommu_hap_pt_share )
{
if ( old_mfn && (old_mfn != mfn_x(mfn)) )
amd_iommu_flush_pages(p2m->domain, gfn, page_order);
}
else
{
if ( p2mt == p2m_ram_rw )
for ( i = 0; i < (1UL << page_order); i++ )
iommu_map_page(p2m->domain, gfn+i, mfn_x(mfn)+i,
IOMMUF_readable|IOMMUF_writable);
else
for ( int i = 0; i < (1UL << page_order); i++ )
iommu_unmap_page(p2m->domain, gfn+i);
}
}
out:
unmap_domain_page(table);
return rc;
}