void __trace_pv_page_fault(unsigned long addr, unsigned error_code)
{
unsigned long eip = guest_cpu_user_regs()->eip;
if ( is_pv_32bit_vcpu(current) )
{
struct __packed {
u32 eip, addr, error_code;
} d;
d.eip = eip;
d.addr = addr;
d.error_code = error_code;
__trace_var(TRC_PV_PAGE_FAULT, 1, sizeof(d), &d);
}
else
{
struct __packed {
unsigned long eip, addr;
u32 error_code;
} d;
unsigned event;
d.eip = eip;
d.addr = addr;
d.error_code = error_code;
event = TRC_PV_PAGE_FAULT;
event |= TRC_64_FLAG;
__trace_var(event, 1, sizeof(d), &d);
}
}
void __trace_trap_one_addr(unsigned event, unsigned long va)
{
if ( is_pv_32bit_vcpu(current) )
{
u32 d = va;
__trace_var(event, 1, sizeof(d), &d);
}
else
{
event |= TRC_64_FLAG;
__trace_var(event, 1, sizeof(va), &va);
}
}
static void decrease_reservation(struct memop_args *a)
{
unsigned long i, j;
xen_pfn_t gmfn;
if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
a->nr_extents-1) ||
a->extent_order > MAX_ORDER )
return;
for ( i = a->nr_done; i < a->nr_extents; i++ )
{
if ( i != a->nr_done && hypercall_preempt_check() )
{
a->preempted = 1;
goto out;
}
if ( unlikely(__copy_from_guest_offset(&gmfn, a->extent_list, i, 1)) )
goto out;
if ( tb_init_done )
{
struct {
u64 gfn;
int d:16,order:16;
} t;
t.gfn = gmfn;
t.d = a->domain->domain_id;
t.order = a->extent_order;
__trace_var(TRC_MEM_DECREASE_RESERVATION, 0, sizeof(t), &t);
}
/* See if populate-on-demand wants to handle this */
if ( is_hvm_domain(a->domain)
&& p2m_pod_decrease_reservation(a->domain, gmfn, a->extent_order) )
continue;
/* With the lack for iommu on some ARM platform, domain with DMA-capable
* device must retrieve the same pfn when the hypercall
* populate_physmap is called.
*/
if ( is_domain_direct_mapped(a->domain) )
continue;
for ( j = 0; j < (1 << a->extent_order); j++ )
if ( !guest_remove_page(a->domain, gmfn + j) )
goto out;
}
out:
a->nr_done = i;
}
void __trace_ptwr_emulation(unsigned long addr, l1_pgentry_t npte)
{
unsigned long eip = guest_cpu_user_regs()->eip;
/* We have a couple of different modes to worry about:
* - 32-on-32: 32-bit pte, 32-bit virtual addresses
* - pae-on-pae, pae-on-64: 64-bit pte, 32-bit virtual addresses
* - 64-on-64: 64-bit pte, 64-bit virtual addresses
* pae-on-64 is the only one that requires extra code; in all other
* cases, "unsigned long" is the size of a guest virtual address.
*/
if ( is_pv_32bit_vcpu(current) )
{
struct __packed {
l1_pgentry_t pte;
u32 addr, eip;
} d;
d.addr = addr;
d.eip = eip;
d.pte = npte;
__trace_var(TRC_PV_PTWR_EMULATION_PAE, 1, sizeof(d), &d);
}
else
{
struct {
l1_pgentry_t pte;
unsigned long addr, eip;
} d;
unsigned event;
d.addr = addr;
d.eip = eip;
d.pte = npte;
event = TRC_PV_PTWR_EMULATION;
event |= TRC_64_FLAG;
__trace_var(event, 1/*tsc*/, sizeof(d), &d);
}
}
static inline void trace_continue_running(struct vcpu *v)
{
struct { uint32_t vcpu:16, domain:16; } d;
if ( likely(!tb_init_done) )
return;
d.vcpu = v->vcpu_id;
d.domain = v->domain->domain_id;
__trace_var(TRC_SCHED_CONTINUE_RUNNING, 1/*tsc*/, sizeof(d), &d);
}
void __trace_pv_trap(int trapnr, unsigned long eip,
int use_error_code, unsigned error_code)
{
if ( is_pv_32bit_vcpu(current) )
{
struct __packed {
unsigned eip:32,
trapnr:15,
use_error_code:1,
error_code:16;
} d;
d.eip = eip;
d.trapnr = trapnr;
d.error_code = error_code;
d.use_error_code=!!use_error_code;
__trace_var(TRC_PV_TRAP, 1, sizeof(d), &d);
}
else
{
struct __packed {
unsigned long eip;
unsigned trapnr:15,
use_error_code:1,
error_code:16;
} d;
unsigned event;
d.eip = eip;
d.trapnr = trapnr;
d.error_code = error_code;
d.use_error_code=!!use_error_code;
event = TRC_PV_TRAP;
event |= TRC_64_FLAG;
__trace_var(event, 1, sizeof(d), &d);
}
}
void __trace_trap_two_addr(unsigned event, unsigned long va1,
unsigned long va2)
{
if ( is_pv_32bit_vcpu(current) )
{
struct __packed {
u32 va1, va2;
} d;
d.va1=va1;
d.va2=va2;
__trace_var(event, 1, sizeof(d), &d);
}
else
{
struct __packed {
unsigned long va1, va2;
} d;
d.va1=va1;
d.va2=va2;
event |= TRC_64_FLAG;
__trace_var(event, 1, sizeof(d), &d);
}
}
static void decrease_reservation(struct memop_args *a)
{
unsigned long i, j;
xen_pfn_t gmfn;
if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
a->nr_extents-1) )
return;
mcd_mem_dec_trap(a->domain, (a->nr_extents - a->nr_done));
for ( i = a->nr_done; i < a->nr_extents; i++ )
{
if ( hypercall_preempt_check() )
{
a->preempted = 1;
goto out;
}
if ( unlikely(__copy_from_guest_offset(&gmfn, a->extent_list, i, 1)) )
goto out;
if ( tb_init_done )
{
struct {
u64 gfn;
int d:16,order:16;
} t;
t.gfn = gmfn;
t.d = a->domain->domain_id;
t.order = a->extent_order;
__trace_var(TRC_MEM_DECREASE_RESERVATION, 0, sizeof(t), &t);
}
/* See if populate-on-demand wants to handle this */
if ( is_hvm_domain(a->domain)
&& p2m_pod_decrease_reservation(a->domain, gmfn, a->extent_order) )
continue;
for ( j = 0; j < (1 << a->extent_order); j++ )
if ( !guest_remove_page(a->domain, gmfn + j) )
goto out;
}
out:
a->nr_done = i;
mcd_mem_upt_trap(a->domain);
}
static inline void trace_runstate_change(struct vcpu *v, int new_state)
{
struct { uint32_t vcpu:16, domain:16; } d;
uint32_t event;
if ( likely(!tb_init_done) )
return;
d.vcpu = v->vcpu_id;
d.domain = v->domain->domain_id;
event = TRC_SCHED_RUNSTATE_CHANGE;
event |= ( v->runstate.state & 0x3 ) << 8;
event |= ( new_state & 0x3 ) << 4;
__trace_var(event, 1/*tsc*/, sizeof(d), (unsigned char *)&d);
}
asmlinkage void trace_hypercall(void)
{
struct cpu_user_regs *regs = guest_cpu_user_regs();
#ifdef __x86_64__
if ( is_pv_32on64_vcpu(current) )
{
struct {
u32 eip,eax;
} __attribute__((packed)) d;
d.eip = regs->eip;
d.eax = regs->eax;
__trace_var(TRC_PV_HYPERCALL, 1, sizeof(d), &d);
}
else
#endif
{
// Returns 0 on error (out of memory)
static int
p2m_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
{
// XXX -- this might be able to be faster iff current->domain == d
mfn_t table_mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
void *table =map_domain_page(mfn_x(table_mfn));
unsigned long i, gfn_remainder = gfn;
l1_pgentry_t *p2m_entry;
l1_pgentry_t entry_content;
l2_pgentry_t l2e_content;
l3_pgentry_t l3e_content;
int rv=0;
unsigned int iommu_pte_flags = (p2mt == p2m_ram_rw) ?
IOMMUF_readable|IOMMUF_writable:
0;
unsigned long old_mfn = 0;
if ( tb_init_done )
{
struct {
u64 gfn, mfn;
int p2mt;
int d:16,order:16;
} t;
t.gfn = gfn;
t.mfn = mfn_x(mfn);
t.p2mt = p2mt;
t.d = p2m->domain->domain_id;
t.order = page_order;
__trace_var(TRC_MEM_SET_P2M_ENTRY, 0, sizeof(t), &t);
}
#if CONFIG_PAGING_LEVELS >= 4
if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L4_PAGETABLE_SHIFT - PAGE_SHIFT,
L4_PAGETABLE_ENTRIES, PGT_l3_page_table) )
goto out;
#endif
/*
* Try to allocate 1GB page table if this feature is supported.
*/
if ( page_order == PAGE_ORDER_1G )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
l3e_content = mfn_valid(mfn)
? l3e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) | _PAGE_PSE)
: l3e_empty();
entry_content.l1 = l3e_content.l3;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 3);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/*
* When using PAE Xen, we only allow 33 bits of pseudo-physical
* address in translated guests (i.e. 8 GBytes). This restriction
* comes from wanting to map the P2M table into the 16MB RO_MPT hole
* in Xen's address space for translated PV guests.
* When using AMD's NPT on PAE Xen, we are restricted to 4GB.
*/
else if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
((CONFIG_PAGING_LEVELS == 3)
? (hap_enabled(p2m->domain) ? 4 : 8)
: L3_PAGETABLE_ENTRIES),
PGT_l2_page_table) )
goto out;
if ( page_order == PAGE_ORDER_4K )
{
if ( !p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES, PGT_l1_page_table) )
goto out;
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
0, L1_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( mfn_valid(mfn) || (p2mt == p2m_mmio_direct)
|| p2m_is_paging(p2mt) )
entry_content = p2m_l1e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn));
else
entry_content = l1e_empty();
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
/* level 1 entry */
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 1);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
}
else if ( page_order == PAGE_ORDER_2M )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
/* FIXME: Deal with 4k replaced by 2meg pages */
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
if ( mfn_valid(mfn) || p2m_is_magic(p2mt) )
l2e_content = l2e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) |
_PAGE_PSE);
else
l2e_content = l2e_empty();
entry_content.l1 = l2e_content.l2;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 2);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/* Track the highest gfn for which we have ever had a valid mapping */
if ( p2mt != p2m_invalid
&& (gfn + (1UL << page_order) - 1 > p2m->max_mapped_pfn) )
p2m->max_mapped_pfn = gfn + (1UL << page_order) - 1;
if ( iommu_enabled && need_iommu(p2m->domain) )
{
if ( iommu_hap_pt_share )
{
if ( old_mfn && (old_mfn != mfn_x(mfn)) )
amd_iommu_flush_pages(p2m->domain, gfn, page_order);
}
else
{
if ( p2mt == p2m_ram_rw )
for ( i = 0; i < (1UL << page_order); i++ )
iommu_map_page(p2m->domain, gfn+i, mfn_x(mfn)+i,
IOMMUF_readable|IOMMUF_writable);
else
for ( int i = 0; i < (1UL << page_order); i++ )
iommu_unmap_page(p2m->domain, gfn+i);
}
}
/* Success */
rv = 1;
out:
unmap_domain_page(table);
return rv;
}
/* Returns: 0 for success, -errno for failure */
static int
p2m_pt_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
{
/* XXX -- this might be able to be faster iff current->domain == d */
mfn_t table_mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
void *table = map_domain_page(mfn_x(table_mfn));
unsigned long i, gfn_remainder = gfn;
l1_pgentry_t *p2m_entry;
l1_pgentry_t entry_content;
l2_pgentry_t l2e_content;
l3_pgentry_t l3e_content;
int rc;
unsigned int iommu_pte_flags = (p2mt == p2m_ram_rw) ?
IOMMUF_readable|IOMMUF_writable:
0;
unsigned long old_mfn = 0;
if ( tb_init_done )
{
struct {
u64 gfn, mfn;
int p2mt;
int d:16,order:16;
} t;
t.gfn = gfn;
t.mfn = mfn_x(mfn);
t.p2mt = p2mt;
t.d = p2m->domain->domain_id;
t.order = page_order;
__trace_var(TRC_MEM_SET_P2M_ENTRY, 0, sizeof(t), &t);
}
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L4_PAGETABLE_SHIFT - PAGE_SHIFT,
L4_PAGETABLE_ENTRIES, PGT_l3_page_table);
if ( rc )
goto out;
/*
* Try to allocate 1GB page table if this feature is supported.
*/
if ( page_order == PAGE_ORDER_1G )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
l3e_content = mfn_valid(mfn)
? l3e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) | _PAGE_PSE)
: l3e_empty();
entry_content.l1 = l3e_content.l3;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 3);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
else
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder,
gfn, L3_PAGETABLE_SHIFT - PAGE_SHIFT,
L3_PAGETABLE_ENTRIES, PGT_l2_page_table);
if ( rc )
goto out;
}
if ( page_order == PAGE_ORDER_4K )
{
rc = p2m_next_level(p2m, &table_mfn, &table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES, PGT_l1_page_table);
if ( rc )
goto out;
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
0, L1_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
if ( mfn_valid(mfn) || (p2mt == p2m_mmio_direct)
|| p2m_is_paging(p2mt) )
entry_content = p2m_l1e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn));
else
entry_content = l1e_empty();
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
/* level 1 entry */
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 1);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
}
else if ( page_order == PAGE_ORDER_2M )
{
l1_pgentry_t old_entry = l1e_empty();
p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
L2_PAGETABLE_SHIFT - PAGE_SHIFT,
L2_PAGETABLE_ENTRIES);
ASSERT(p2m_entry);
/* FIXME: Deal with 4k replaced by 2meg pages */
if ( (l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) &&
!(l1e_get_flags(*p2m_entry) & _PAGE_PSE) )
{
/* We're replacing a non-SP page with a superpage. Make sure to
* handle freeing the table properly. */
old_entry = *p2m_entry;
}
ASSERT(!mfn_valid(mfn) || p2mt != p2m_mmio_direct);
if ( mfn_valid(mfn) || p2m_is_pod(p2mt) )
l2e_content = l2e_from_pfn(mfn_x(mfn),
p2m_type_to_flags(p2mt, mfn) |
_PAGE_PSE);
else
l2e_content = l2e_empty();
entry_content.l1 = l2e_content.l2;
if ( entry_content.l1 != 0 )
{
p2m_add_iommu_flags(&entry_content, 0, iommu_pte_flags);
old_mfn = l1e_get_pfn(*p2m_entry);
}
p2m->write_p2m_entry(p2m, gfn, p2m_entry, table_mfn, entry_content, 2);
/* NB: paging_write_p2m_entry() handles tlb flushes properly */
/* Free old intermediate tables if necessary */
if ( l1e_get_flags(old_entry) & _PAGE_PRESENT )
p2m_free_entry(p2m, &old_entry, page_order);
}
/* Track the highest gfn for which we have ever had a valid mapping */
if ( p2mt != p2m_invalid
&& (gfn + (1UL << page_order) - 1 > p2m->max_mapped_pfn) )
p2m->max_mapped_pfn = gfn + (1UL << page_order) - 1;
if ( iommu_enabled && need_iommu(p2m->domain) )
{
if ( iommu_hap_pt_share )
{
if ( old_mfn && (old_mfn != mfn_x(mfn)) )
amd_iommu_flush_pages(p2m->domain, gfn, page_order);
}
else
{
if ( p2mt == p2m_ram_rw )
for ( i = 0; i < (1UL << page_order); i++ )
iommu_map_page(p2m->domain, gfn+i, mfn_x(mfn)+i,
IOMMUF_readable|IOMMUF_writable);
else
for ( int i = 0; i < (1UL << page_order); i++ )
iommu_unmap_page(p2m->domain, gfn+i);
}
}
out:
unmap_domain_page(table);
return rc;
}
