/* Atomically look up a GFN and take a reference count on the backing page. */
struct page_info *get_page_from_gfn_p2m(
struct domain *d, struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q)
{
struct page_info *page = NULL;
p2m_access_t _a;
p2m_type_t _t;
mfn_t mfn;
/* Allow t or a to be NULL */
t = t ?: &_t;
a = a ?: &_a;
if ( likely(!p2m_locked_by_me(p2m)) )
{
/* Fast path: look up and get out */
p2m_read_lock(p2m);
mfn = __get_gfn_type_access(p2m, gfn, t, a, 0, NULL, 0);
if ( (p2m_is_ram(*t) || p2m_is_grant(*t))
&& mfn_valid(mfn)
&& !((q & P2M_UNSHARE) && p2m_is_shared(*t)) )
{
page = mfn_to_page(mfn);
if ( !get_page(page, d)
/* Page could be shared */
&& !get_page(page, dom_cow) )
page = NULL;
}
p2m_read_unlock(p2m);
if ( page )
return page;
/* Error path: not a suitable GFN at all */
if ( !p2m_is_ram(*t) && !p2m_is_paging(*t) && !p2m_is_pod(*t) )
return NULL;
}
/* Slow path: take the write lock and do fixups */
mfn = get_gfn_type_access(p2m, gfn, t, a, q, NULL);
if ( p2m_is_ram(*t) && mfn_valid(mfn) )
{
page = mfn_to_page(mfn);
if ( !get_page(page, d) )
page = NULL;
}
put_gfn(d, gfn);
return page;
}
/* Returns: 0 for success, -errno for failure */
static int
p2m_pt_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
{
/* XXX -- this might be able to be faster iff current->domain == d */
mfn_t table_mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
void *table = map_domain_page(mfn_x(table_mfn));
unsigned long i, gfn_remainder = gfn;
l1_pgentry_t *p2m_entry;
l1_pgentry_t entry_content;
l2_pgentry_t l2e_content;
l3_pgentry_t l3e_content;
int rc;
unsigned int iommu_pte_flags = (p2mt == p2m_ram_rw) ?
IOMMUF_readable|IOMMUF_writable:
0;
unsigned long old_mfn = 0;
if ( tb_init_done )
{
struct {
u64 gfn, mfn;
int p2mt;
int d:16,order:16;
} t;
t.gfn = gfn;
t.mfn = mfn_x(mfn);
t.p2mt = p2mt;
t.d = p2m->domain->domain_id;
t.order = page_order;
__trace_var(TRC_MEM_SET_P2M_ENTRY, 0, sizeof(t), &t);
}
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L4_PAGETABLE_SHIFT - PAGE_SHIFT,
L4_PAGETABLE_ENTRIES, PGT_l3_page_table);
if ( rc )
goto out;
/*
* Try to allocate 1GB page table if this feature is supported.
*/
if ( page_order == PAGE_ORDER_1G )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
l3e_content = mfn_valid(mfn)
? l3e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) | _PAGE_PSE)
: l3e_empty();
entry_content.l1 = l3e_content.l3;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 3);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
else
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder,
gfn, L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES, PGT_l2_page_table);
if ( rc )
goto out;
}
if ( page_order == PAGE_ORDER_4K )
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES, PGT_l1_page_table);
if ( rc )
goto out;
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
0, L1_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( mfn_valid(mfn) || (p2mt == p2m_mmio_direct)
|| p2m_is_paging(p2mt) )
entry_content = p2m_l1e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn));
else
entry_content = l1e_empty();
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
/* level 1 entry */
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 1);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
}
else if ( page_order == PAGE_ORDER_2M )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
/* FIXME: Deal with 4k replaced by 2meg pages */
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
if ( mfn_valid(mfn) || p2m_is_pod(p2mt) )
l2e_content = l2e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) |
_PAGE_PSE);
else
l2e_content = l2e_empty();
entry_content.l1 = l2e_content.l2;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 2);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/* Track the highest gfn for which we have ever had a valid mapping */
if ( p2mt != p2m_invalid
&& (gfn + (1UL << page_order) - 1 > p2m->max_mapped_pfn) )
p2m->max_mapped_pfn = gfn + (1UL << page_order) - 1;
if ( iommu_enabled && need_iommu(p2m->domain) )
{
if ( iommu_hap_pt_share )
{
if ( old_mfn && (old_mfn != mfn_x(mfn)) )
amd_iommu_flush_pages(p2m->domain, gfn, page_order);
}
else
{
if ( p2mt == p2m_ram_rw )
for ( i = 0; i < (1UL << page_order); i++ )
iommu_map_page(p2m->domain, gfn+i, mfn_x(mfn)+i,
IOMMUF_readable|IOMMUF_writable);
else
for ( int i = 0; i < (1UL << page_order); i++ )
iommu_unmap_page(p2m->domain, gfn+i);
}
}
out:
unmap_domain_page(table);
return rc;
}
