static int vtimer_emulate_cp64(struct cpu_user_regs *regs, union hsr hsr)
{
struct hsr_cp64 cp64 = hsr.cp64;
uint32_t *r1 = (uint32_t *)select_user_reg(regs, cp64.reg1);
uint32_t *r2 = (uint32_t *)select_user_reg(regs, cp64.reg2);
uint64_t x = (uint64_t)(*r1) | ((uint64_t)(*r2) << 32);
if ( cp64.read )
perfc_incr(vtimer_cp64_reads);
else
perfc_incr(vtimer_cp64_writes);
switch ( hsr.bits & HSR_CP64_REGS_MASK )
{
case HSR_CPREG64(CNTP_CVAL):
if ( !vtimer_cntp_cval(regs, &x, cp64.read) )
return 0;
break;
default:
return 0;
}
if ( cp64.read )
{
*r1 = (uint32_t)(x & 0xffffffff);
*r2 = (uint32_t)(x >> 32);
}
static void phys_timer_expired(void *data)
{
struct vtimer *t = data;
t->ctl |= CNTx_CTL_PENDING;
if ( !(t->ctl & CNTx_CTL_MASK) )
{
perfc_incr(vtimer_phys_inject);
vgic_vcpu_inject_irq(t->v, t->irq);
}
else
perfc_incr(vtimer_phys_masked);
}
static void virt_timer_expired(void *data)
{
struct vtimer *t = data;
t->ctl |= CNTx_CTL_MASK;
vgic_vcpu_inject_irq(t->v, t->irq);
perfc_incr(vtimer_virt_inject);
}
int rdmsr_viridian_regs(uint32_t idx, uint64_t *val)
{
struct vcpu *v = current;
struct domain *d = v->domain;
if ( !is_viridian_domain(d) )
return 0;
switch ( idx )
{
case VIRIDIAN_MSR_GUEST_OS_ID:
perfc_incr(mshv_rdmsr_osid);
*val = d->arch.hvm_domain.viridian.guest_os_id.raw;
break;
case VIRIDIAN_MSR_HYPERCALL:
perfc_incr(mshv_rdmsr_hc_page);
*val = d->arch.hvm_domain.viridian.hypercall_gpa.raw;
break;
case VIRIDIAN_MSR_VP_INDEX:
perfc_incr(mshv_rdmsr_vp_index);
*val = v->vcpu_id;
break;
case VIRIDIAN_MSR_TSC_FREQUENCY:
perfc_incr(mshv_rdmsr_tsc_frequency);
*val = (uint64_t)d->arch.tsc_khz * 1000ull;
break;
case VIRIDIAN_MSR_APIC_FREQUENCY:
perfc_incr(mshv_rdmsr_apic_frequency);
*val = 1000000000ull / APIC_BUS_CYCLE_NS;
break;
case VIRIDIAN_MSR_ICR:
perfc_incr(mshv_rdmsr_icr);
*val = (((uint64_t)vlapic_get_reg(vcpu_vlapic(v), APIC_ICR2) << 32) |
vlapic_get_reg(vcpu_vlapic(v), APIC_ICR));
break;
case VIRIDIAN_MSR_TPR:
perfc_incr(mshv_rdmsr_tpr);
*val = vlapic_get_reg(vcpu_vlapic(v), APIC_TASKPRI);
break;
case VIRIDIAN_MSR_APIC_ASSIST:
perfc_incr(mshv_rdmsr_apic_msr);
*val = v->arch.hvm_vcpu.viridian.apic_assist.raw;
break;
default:
return 0;
}
return 1;
}
static void realmode_emulate_one(struct hvm_emulate_ctxt *hvmemul_ctxt)
{
struct vcpu *curr = current;
int rc;
perfc_incr(realmode_emulations);
rc = hvm_emulate_one(hvmemul_ctxt);
if ( rc == X86EMUL_UNHANDLEABLE )
{
gdprintk(XENLOG_ERR, "Failed to emulate insn.\n");
goto fail;
}
if ( rc == X86EMUL_EXCEPTION )
{
if ( !hvmemul_ctxt->exn_pending )
{
unsigned long intr_info;
__vmread(VM_ENTRY_INTR_INFO, &intr_info);
__vmwrite(VM_ENTRY_INTR_INFO, 0);
if ( !(intr_info & INTR_INFO_VALID_MASK) )
{
gdprintk(XENLOG_ERR, "Exception pending but no info.\n");
goto fail;
}
hvmemul_ctxt->trap.vector = (uint8_t)intr_info;
hvmemul_ctxt->trap.insn_len = 0;
}
if ( unlikely(curr->domain->debugger_attached) &&
((hvmemul_ctxt->trap.vector == TRAP_debug) ||
(hvmemul_ctxt->trap.vector == TRAP_int3)) )
{
domain_pause_for_debugger();
}
else if ( curr->arch.hvm_vcpu.guest_cr[0] & X86_CR0_PE )
{
gdprintk(XENLOG_ERR, "Exception %02x in protected mode.\n",
hvmemul_ctxt->trap.vector);
goto fail;
}
else
{
realmode_deliver_exception(
hvmemul_ctxt->trap.vector,
hvmemul_ctxt->trap.insn_len,
hvmemul_ctxt);
}
}
return;
fail:
hvm_dump_emulation_state(XENLOG_G_ERR "Real-mode", hvmemul_ctxt);
domain_crash(curr->domain);
}
fastcall void smp_apic_timer_interrupt(struct cpu_user_regs * regs)
{
struct cpu_user_regs *old_regs = set_irq_regs(regs);
ack_APIC_irq();
perfc_incr(apic_timer);
raise_softirq(TIMER_SOFTIRQ);
set_irq_regs(old_regs);
}
static bool vtimer_emulate_cp64(struct cpu_user_regs *regs, union hsr hsr)
{
struct hsr_cp64 cp64 = hsr.cp64;
if ( cp64.read )
perfc_incr(vtimer_cp64_reads);
else
perfc_incr(vtimer_cp64_writes);
switch ( hsr.bits & HSR_CP64_REGS_MASK )
{
case HSR_CPREG64(CNTP_CVAL):
return vreg_emulate_cp64(regs, hsr, vtimer_cntp_cval);
default:
return false;
}
}
void invalidate_interrupt(struct cpu_user_regs *regs)
{
ack_APIC_irq();
perfc_incr(ipis);
if ( !__sync_local_execstate() ||
(flush_flags & (FLUSH_TLB_GLOBAL | FLUSH_CACHE)) )
flush_area_local(flush_va, flush_flags);
cpumask_clear_cpu(smp_processor_id(), &flush_cpumask);
}
int rdmsr_viridian_regs(uint32_t idx, uint64_t *val)
{
struct vcpu *v = current;
if ( !is_viridian_domain(v->domain) )
return 0;
switch ( idx )
{
case VIRIDIAN_MSR_GUEST_OS_ID:
perfc_incr(mshv_rdmsr_osid);
*val = v->domain->arch.hvm_domain.viridian.guest_os_id.raw;
break;
case VIRIDIAN_MSR_HYPERCALL:
perfc_incr(mshv_rdmsr_hc_page);
*val = v->domain->arch.hvm_domain.viridian.hypercall_gpa.raw;
break;
case VIRIDIAN_MSR_VP_INDEX:
perfc_incr(mshv_rdmsr_vp_index);
*val = v->vcpu_id;
break;
case VIRIDIAN_MSR_ICR:
perfc_incr(mshv_rdmsr_icr);
*val = (((uint64_t)vlapic_get_reg(vcpu_vlapic(v), APIC_ICR2) << 32) |
vlapic_get_reg(vcpu_vlapic(v), APIC_ICR));
break;
case VIRIDIAN_MSR_TPR:
perfc_incr(mshv_rdmsr_tpr);
*val = vlapic_get_reg(vcpu_vlapic(v), APIC_TASKPRI);
break;
default:
return 0;
}
return 1;
}
static bool vtimer_emulate_cp32(struct cpu_user_regs *regs, union hsr hsr)
{
struct hsr_cp32 cp32 = hsr.cp32;
if ( cp32.read )
perfc_incr(vtimer_cp32_reads);
else
perfc_incr(vtimer_cp32_writes);
switch ( hsr.bits & HSR_CP32_REGS_MASK )
{
case HSR_CPREG32(CNTP_CTL):
return vreg_emulate_cp32(regs, hsr, vtimer_cntp_ctl);
case HSR_CPREG32(CNTP_TVAL):
return vreg_emulate_cp32(regs, hsr, vtimer_cntp_tval);
default:
return false;
}
}
static int vtimer_emulate_cp32(struct cpu_user_regs *regs, union hsr hsr)
{
struct hsr_cp32 cp32 = hsr.cp32;
/*
* Initialize to zero to avoid leaking data if there is an
* implementation error in the emulation (such as not correctly
* setting r).
*/
uint32_t r = 0;
int res;
if ( cp32.read )
perfc_incr(vtimer_cp32_reads);
else
perfc_incr(vtimer_cp32_writes);
if ( !cp32.read )
r = get_user_reg(regs, cp32.reg);
switch ( hsr.bits & HSR_CP32_REGS_MASK )
{
case HSR_CPREG32(CNTP_CTL):
res = vtimer_cntp_ctl(regs, &r, cp32.read);
break;
case HSR_CPREG32(CNTP_TVAL):
res = vtimer_cntp_tval(regs, &r, cp32.read);
break;
default:
return 0;
}
if ( res && cp32.read )
set_user_reg(regs, cp32.reg, r);
return res;
}
/* Handle the firing timer */
static void timer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
if ( irq == (timer_irq[TIMER_HYP_PPI]) &&
READ_SYSREG32(CNTHP_CTL_EL2) & CNTx_CTL_PENDING )
{
perfc_incr(hyp_timer_irqs);
/* Signal the generic timer code to do its work */
raise_softirq(TIMER_SOFTIRQ);
/* Disable the timer to avoid more interrupts */
WRITE_SYSREG32(0, CNTHP_CTL_EL2);
}
if ( irq == (timer_irq[TIMER_PHYS_NONSECURE_PPI]) &&
READ_SYSREG32(CNTP_CTL_EL0) & CNTx_CTL_PENDING )
{
perfc_incr(phys_timer_irqs);
/* Signal the generic timer code to do its work */
raise_softirq(TIMER_SOFTIRQ);
/* Disable the timer to avoid more interrupts */
WRITE_SYSREG32(0, CNTP_CTL_EL0);
}
}
static int vtimer_emulate_cp32(struct cpu_user_regs *regs, union hsr hsr)
{
struct hsr_cp32 cp32 = hsr.cp32;
uint32_t *r = (uint32_t *)select_user_reg(regs, cp32.reg);
if ( cp32.read )
perfc_incr(vtimer_cp32_reads);
else
perfc_incr(vtimer_cp32_writes);
switch ( hsr.bits & HSR_CP32_REGS_MASK )
{
case HSR_CPREG32(CNTP_CTL):
return vtimer_cntp_ctl(regs, r, cp32.read);
case HSR_CPREG32(CNTP_TVAL):
return vtimer_cntp_tval(regs, r, cp32.read);
default:
return 0;
}
}
static bool vtimer_emulate_sysreg(struct cpu_user_regs *regs, union hsr hsr)
{
struct hsr_sysreg sysreg = hsr.sysreg;
if ( sysreg.read )
perfc_incr(vtimer_sysreg_reads);
else
perfc_incr(vtimer_sysreg_writes);
switch ( hsr.bits & HSR_SYSREG_REGS_MASK )
{
case HSR_SYSREG_CNTP_CTL_EL0:
return vreg_emulate_sysreg32(regs, hsr, vtimer_cntp_ctl);
case HSR_SYSREG_CNTP_TVAL_EL0:
return vreg_emulate_sysreg32(regs, hsr, vtimer_cntp_tval);
case HSR_SYSREG_CNTP_CVAL_EL0:
return vreg_emulate_sysreg64(regs, hsr, vtimer_cntp_cval);
default:
return false;
}
}
static int vuart_mmio_write(struct vcpu *v, mmio_info_t *info, void *priv)
{
struct domain *d = v->domain;
struct hsr_dabt dabt = info->dabt;
struct cpu_user_regs *regs = guest_cpu_user_regs();
register_t *r = select_user_reg(regs, dabt.reg);
paddr_t offset = info->gpa - d->arch.vuart.info->base_addr;
perfc_incr(vuart_writes);
if ( offset == d->arch.vuart.info->data_off )
/* ignore any status bits */
vuart_print_char(v, *r & 0xFF);
return 1;
}
static int vuart_mmio_read(struct vcpu *v, mmio_info_t *info, void *priv)
{
struct domain *d = v->domain;
struct hsr_dabt dabt = info->dabt;
struct cpu_user_regs *regs = guest_cpu_user_regs();
register_t *r = select_user_reg(regs, dabt.reg);
paddr_t offset = info->gpa - d->arch.vuart.info->base_addr;
perfc_incr(vuart_reads);
/* By default zeroed the register */
*r = 0;
if ( offset == d->arch.vuart.info->status_off )
/* All holding registers empty, ready to send etc */
*r = d->arch.vuart.info->status;
return 1;
}
static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
/*
* Edge-triggered interrupts can be used for the virtual timer. Even
* if the timer output signal is masked in the context switch, the
* GIC will keep track that of any interrupts raised while IRQS are
* disabled. As soon as IRQs are re-enabled, the virtual interrupt
* will be injected to Xen.
*
* If an IDLE vCPU was scheduled next then we should ignore the
* interrupt.
*/
if ( unlikely(is_idle_vcpu(current)) )
return;
perfc_incr(virt_timer_irqs);
current->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(current->arch.virt_timer.ctl | CNTx_CTL_MASK, CNTV_CTL_EL0);
vgic_vcpu_inject_irq(current, current->arch.virt_timer.irq);
}
int get_page_type(struct page_info *page, unsigned long type)
{
unsigned long nx, x, y = page->u.inuse.type_info;
ASSERT(!(type & ~PGT_type_mask));
again:
do {
x = y;
nx = x + 1;
if ( unlikely((nx & PGT_count_mask) == 0) )
{
MEM_LOG("Type count overflow on pfn %lx", page_to_mfn(page));
return 0;
}
else if ( unlikely((x & PGT_count_mask) == 0) )
{
if ( (x & PGT_type_mask) != type )
{
/*
* On type change we check to flush stale TLB entries. This
* may be unnecessary (e.g., page was GDT/LDT) but those
* circumstances should be very rare.
*/
cpumask_t mask =
page_get_owner(page)->domain_dirty_cpumask;
tlbflush_filter(mask, page->tlbflush_timestamp);
if ( unlikely(!cpus_empty(mask)) )
{
perfc_incr(need_flush_tlb_flush);
flush_tlb_mask(mask);
}
/* We lose existing type, back pointer, and validity. */
nx &= ~(PGT_type_mask | PGT_validated);
nx |= type;
/* No special validation needed for writable pages. */
/* Page tables and GDT/LDT need to be scanned for validity. */
if ( type == PGT_writable_page )
nx |= PGT_validated;
}
}
else if ( unlikely((x & PGT_type_mask) != type) )
{
return 0;
}
else if ( unlikely(!(x & PGT_validated)) )
{
/* Someone else is updating validation of this page. Wait... */
while ( (y = page->u.inuse.type_info) == x )
cpu_relax();
goto again;
}
}
while ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x) );
if ( unlikely(!(nx & PGT_validated)) )
{
/* Noone else is updating simultaneously. */
__set_bit(_PGT_validated, &page->u.inuse.type_info);
}
return 1;
}
/* Dispatch an interrupt */
void do_IRQ(struct cpu_user_regs *regs, unsigned int irq, int is_fiq)
{
struct irq_desc *desc = irq_to_desc(irq);
perfc_incr(irqs);
ASSERT(irq >= 16); /* SGIs do not come down this path */
if (irq < 32)
perfc_incr(ppis);
else
perfc_incr(spis);
/* TODO: this_cpu(irq_count)++; */
irq_enter();
spin_lock(&desc->lock);
desc->handler->ack(desc);
if ( !desc->action )
{
printk("Unknown %s %#3.3x\n",
is_fiq ? "FIQ" : "IRQ", irq);
goto out;
}
if ( test_bit(_IRQ_GUEST, &desc->status) )
{
struct irq_guest *info = irq_get_guest_info(desc);
perfc_incr(guest_irqs);
desc->handler->end(desc);
set_bit(_IRQ_INPROGRESS, &desc->status);
/*
* The irq cannot be a PPI, we only support delivery of SPIs to
* guests.
*/
vgic_vcpu_inject_spi(info->d, info->virq);
goto out_no_end;
}
set_bit(_IRQ_PENDING, &desc->status);
/*
* Since we set PENDING, if another processor is handling a different
* instance of this same irq, the other processor will take care of it.
*/
if ( test_bit(_IRQ_DISABLED, &desc->status) ||
test_bit(_IRQ_INPROGRESS, &desc->status) )
goto out;
set_bit(_IRQ_INPROGRESS, &desc->status);
while ( test_bit(_IRQ_PENDING, &desc->status) )
{
struct irqaction *action;
clear_bit(_IRQ_PENDING, &desc->status);
action = desc->action;
spin_unlock_irq(&desc->lock);
do
{
action->handler(irq, action->dev_id, regs);
action = action->next;
} while ( action );
spin_lock_irq(&desc->lock);
}
clear_bit(_IRQ_INPROGRESS, &desc->status);
out:
desc->handler->end(desc);
out_no_end:
spin_unlock(&desc->lock);
irq_exit();
}
void call_function_interrupt(struct cpu_user_regs *regs)
{
ack_APIC_irq();
perfc_incr(ipis);
smp_call_function_interrupt();
}
IA64FAULT
ia64_hypercall(struct pt_regs *regs)
{
struct vcpu *v = current;
struct sal_ret_values x;
efi_status_t efi_ret_value;
fpswa_ret_t fpswa_ret;
IA64FAULT fault;
unsigned long index = regs->r2 & FW_HYPERCALL_NUM_MASK_HIGH;
perfc_incra(fw_hypercall, index >> 8);
switch (index) {
case FW_HYPERCALL_XEN:
return xen_hypercall(regs);
case FW_HYPERCALL_XEN_FAST:
return xen_fast_hypercall(regs);
case FW_HYPERCALL_PAL_CALL:
//printk("*** PAL hypercall: index=%d\n",regs->r28);
//FIXME: This should call a C routine
#if 0
// This is very conservative, but avoids a possible
// (and deadly) freeze in paravirtualized domains due
// to a yet-to-be-found bug where pending_interruption
// is zero when it shouldn't be. Since PAL is called
// in the idle loop, this should resolve it
VCPU(v,pending_interruption) = 1;
#endif
if (regs->r28 == PAL_HALT_LIGHT) {
if (vcpu_deliverable_interrupts(v) ||
event_pending(v)) {
perfc_incr(idle_when_pending);
vcpu_pend_unspecified_interrupt(v);
//printk("idle w/int#%d pending!\n",pi);
//this shouldn't happen, but it apparently does quite a bit! so don't
//allow it to happen... i.e. if a domain has an interrupt pending and
//it tries to halt itself because it thinks it is idle, just return here
//as deliver_pending_interrupt is called on the way out and will deliver it
}
else {
perfc_incr(pal_halt_light);
migrate_timer(&v->arch.hlt_timer,
v->processor);
set_timer(&v->arch.hlt_timer,
vcpu_get_next_timer_ns(v));
do_sched_op_compat(SCHEDOP_block, 0);
/* do_block only pends a softirq */
do_softirq();
stop_timer(&v->arch.hlt_timer);
/* do_block() calls
* local_event_delivery_enable(),
* but PAL CALL must be called with
* psr.i = 0 and psr.i is unchanged.
* SDM vol.2 Part I 11.10.2
* PAL Calling Conventions.
*/
local_event_delivery_disable();
}
regs->r8 = 0;
regs->r9 = 0;
regs->r10 = 0;
regs->r11 = 0;
}
else {
struct ia64_pal_retval y;
if (regs->r28 >= PAL_COPY_PAL)
y = xen_pal_emulator
(regs->r28, vcpu_get_gr (v, 33),
vcpu_get_gr (v, 34),
vcpu_get_gr (v, 35));
else
y = xen_pal_emulator(regs->r28,regs->r29,
regs->r30,regs->r31);
regs->r8 = y.status; regs->r9 = y.v0;
regs->r10 = y.v1; regs->r11 = y.v2;
}
break;
case FW_HYPERCALL_SAL_CALL:
x = sal_emulator(vcpu_get_gr(v,32),vcpu_get_gr(v,33),
vcpu_get_gr(v,34),vcpu_get_gr(v,35),
vcpu_get_gr(v,36),vcpu_get_gr(v,37),
vcpu_get_gr(v,38),vcpu_get_gr(v,39));
regs->r8 = x.r8; regs->r9 = x.r9;
regs->r10 = x.r10; regs->r11 = x.r11;
break;
case FW_HYPERCALL_SAL_RETURN:
if ( !test_and_set_bit(_VPF_down, &v->pause_flags) )
vcpu_sleep_nosync(v);
break;
case FW_HYPERCALL_EFI_CALL:
efi_ret_value = efi_emulator (regs, &fault);
if (fault != IA64_NO_FAULT) return fault;
regs->r8 = efi_ret_value;
break;
case FW_HYPERCALL_IPI:
fw_hypercall_ipi (regs);
break;
case FW_HYPERCALL_SET_SHARED_INFO_VA:
regs->r8 = domain_set_shared_info_va (regs->r28);
break;
case FW_HYPERCALL_FPSWA_BASE:
switch (regs->r2) {
case FW_HYPERCALL_FPSWA_BROKEN:
gdprintk(XENLOG_WARNING,
"Old fpswa hypercall was called (0x%lx).\n"
"Please update your domain builder. ip 0x%lx\n",
FW_HYPERCALL_FPSWA_BROKEN, regs->cr_iip);
fpswa_ret = fw_hypercall_fpswa_error();
break;
case FW_HYPERCALL_FPSWA:
fpswa_ret = fw_hypercall_fpswa(v, regs);
break;
default:
gdprintk(XENLOG_ERR, "unknown fpswa hypercall %lx\n",
regs->r2);
fpswa_ret = fw_hypercall_fpswa_error();
break;
}
regs->r8 = fpswa_ret.status;
regs->r9 = fpswa_ret.err0;
regs->r10 = fpswa_ret.err1;
regs->r11 = fpswa_ret.err2;
break;
case __HYPERVISOR_opt_feature:
{
XEN_GUEST_HANDLE(void) arg;
struct xen_ia64_opt_feature optf;
set_xen_guest_handle(arg, (void*)(vcpu_get_gr(v, 32)));
if (copy_from_guest(&optf, arg, 1) == 0)
regs->r8 = domain_opt_feature(v->domain, &optf);
else
regs->r8 = -EFAULT;
break;
}
case FW_HYPERCALL_SIOEMU:
sioemu_hypercall(regs);
break;
default:
printk("unknown ia64 fw hypercall %lx\n", regs->r2);
regs->r8 = do_ni_hypercall();
}
return IA64_NO_FAULT;
}
static void realmode_emulate_one(struct hvm_emulate_ctxt *hvmemul_ctxt)
{
struct vcpu *curr = current;
uint32_t intr_info;
int rc;
perfc_incr(realmode_emulations);
rc = hvm_emulate_one(hvmemul_ctxt);
if ( rc == X86EMUL_UNHANDLEABLE )
{
gdprintk(XENLOG_ERR, "Failed to emulate insn.\n");
goto fail;
}
if ( rc == X86EMUL_EXCEPTION )
{
if ( !hvmemul_ctxt->exn_pending )
{
intr_info = __vmread(VM_ENTRY_INTR_INFO);
__vmwrite(VM_ENTRY_INTR_INFO, 0);
if ( !(intr_info & INTR_INFO_VALID_MASK) )
{
gdprintk(XENLOG_ERR, "Exception pending but no info.\n");
goto fail;
}
hvmemul_ctxt->exn_vector = (uint8_t)intr_info;
hvmemul_ctxt->exn_insn_len = 0;
}
if ( unlikely(curr->domain->debugger_attached) &&
((hvmemul_ctxt->exn_vector == TRAP_debug) ||
(hvmemul_ctxt->exn_vector == TRAP_int3)) )
{
domain_pause_for_debugger();
}
else if ( curr->arch.hvm_vcpu.guest_cr[0] & X86_CR0_PE )
{
gdprintk(XENLOG_ERR, "Exception %02x in protected mode.\n",
hvmemul_ctxt->exn_vector);
goto fail;
}
else
{
realmode_deliver_exception(
hvmemul_ctxt->exn_vector,
hvmemul_ctxt->exn_insn_len,
hvmemul_ctxt);
}
}
return;
fail:
gdprintk(XENLOG_ERR,
"Real-mode emulation failed @ %04x:%08lx: "
"%02x %02x %02x %02x %02x %02x\n",
hvmemul_get_seg_reg(x86_seg_cs, hvmemul_ctxt)->sel,
hvmemul_ctxt->insn_buf_eip,
hvmemul_ctxt->insn_buf[0], hvmemul_ctxt->insn_buf[1],
hvmemul_ctxt->insn_buf[2], hvmemul_ctxt->insn_buf[3],
hvmemul_ctxt->insn_buf[4], hvmemul_ctxt->insn_buf[5]);
domain_crash(curr->domain);
}
void pv_hypercall(struct cpu_user_regs *regs)
{
struct vcpu *curr = current;
unsigned long eax;
ASSERT(guest_kernel_mode(curr, regs));
eax = is_pv_32bit_vcpu(curr) ? regs->eax : regs->rax;
BUILD_BUG_ON(ARRAY_SIZE(pv_hypercall_table) >
ARRAY_SIZE(hypercall_args_table));
if ( (eax >= ARRAY_SIZE(pv_hypercall_table)) ||
!pv_hypercall_table[eax].native )
{
regs->rax = -ENOSYS;
return;
}
curr->hcall_preempted = false;
if ( !is_pv_32bit_vcpu(curr) )
{
unsigned long rdi = regs->rdi;
unsigned long rsi = regs->rsi;
unsigned long rdx = regs->rdx;
unsigned long r10 = regs->r10;
unsigned long r8 = regs->r8;
unsigned long r9 = regs->r9;
#ifndef NDEBUG
/* Deliberately corrupt parameter regs not used by this hypercall. */
switch ( hypercall_args_table[eax].native )
{
case 0: rdi = 0xdeadbeefdeadf00dUL;
case 1: rsi = 0xdeadbeefdeadf00dUL;
case 2: rdx = 0xdeadbeefdeadf00dUL;
case 3: r10 = 0xdeadbeefdeadf00dUL;
case 4: r8 = 0xdeadbeefdeadf00dUL;
case 5: r9 = 0xdeadbeefdeadf00dUL;
}
#endif
if ( unlikely(tb_init_done) )
{
unsigned long args[6] = { rdi, rsi, rdx, r10, r8, r9 };
__trace_hypercall(TRC_PV_HYPERCALL_V2, eax, args);
}
regs->rax = pv_hypercall_table[eax].native(rdi, rsi, rdx, r10, r8, r9);
#ifndef NDEBUG
if ( !curr->hcall_preempted )
{
/* Deliberately corrupt parameter regs used by this hypercall. */
switch ( hypercall_args_table[eax].native )
{
case 6: regs->r9 = 0xdeadbeefdeadf00dUL;
case 5: regs->r8 = 0xdeadbeefdeadf00dUL;
case 4: regs->r10 = 0xdeadbeefdeadf00dUL;
case 3: regs->rdx = 0xdeadbeefdeadf00dUL;
case 2: regs->rsi = 0xdeadbeefdeadf00dUL;
case 1: regs->rdi = 0xdeadbeefdeadf00dUL;
}
}
#endif
}
else
{
unsigned int ebx = regs->ebx;
unsigned int ecx = regs->ecx;
unsigned int edx = regs->edx;
unsigned int esi = regs->esi;
unsigned int edi = regs->edi;
unsigned int ebp = regs->ebp;
#ifndef NDEBUG
/* Deliberately corrupt parameter regs not used by this hypercall. */
switch ( hypercall_args_table[eax].compat )
{
case 0: ebx = 0xdeadf00d;
case 1: ecx = 0xdeadf00d;
case 2: edx = 0xdeadf00d;
case 3: esi = 0xdeadf00d;
case 4: edi = 0xdeadf00d;
case 5: ebp = 0xdeadf00d;
}
#endif
if ( unlikely(tb_init_done) )
{
unsigned long args[6] = { ebx, ecx, edx, esi, edi, ebp };
__trace_hypercall(TRC_PV_HYPERCALL_V2, eax, args);
}
curr->hcall_compat = true;
regs->eax = pv_hypercall_table[eax].compat(ebx, ecx, edx, esi, edi, ebp);
curr->hcall_compat = false;
#ifndef NDEBUG
if ( !curr->hcall_preempted )
{
/* Deliberately corrupt parameter regs used by this hypercall. */
switch ( hypercall_args_table[eax].compat )
{
case 6: regs->ebp = 0xdeadf00d;
case 5: regs->edi = 0xdeadf00d;
case 4: regs->esi = 0xdeadf00d;
case 3: regs->edx = 0xdeadf00d;
case 2: regs->ecx = 0xdeadf00d;
case 1: regs->ebx = 0xdeadf00d;
}
}
#endif
}
/*
* PV guests use SYSCALL or INT $0x82 to make a hypercall, both of which
* have trap semantics. If the hypercall has been preempted, rewind the
* instruction pointer to reexecute the instruction.
*/
if ( curr->hcall_preempted )
regs->rip -= 2;
perfc_incr(hypercalls);
}
int viridian_hypercall(struct cpu_user_regs *regs)
{
struct vcpu *curr = current;
struct domain *currd = curr->domain;
int mode = hvm_guest_x86_mode(curr);
unsigned long input_params_gpa, output_params_gpa;
uint16_t status = HV_STATUS_SUCCESS;
union hypercall_input {
uint64_t raw;
struct {
uint16_t call_code;
uint16_t fast:1;
uint16_t rsvd1:15;
uint16_t rep_count:12;
uint16_t rsvd2:4;
uint16_t rep_start:12;
uint16_t rsvd3:4;
};
} input;
union hypercall_output {
uint64_t raw;
struct {
uint16_t result;
uint16_t rsvd1;
uint32_t rep_complete:12;
uint32_t rsvd2:20;
};
} output = { 0 };
ASSERT(is_viridian_domain(currd));
switch ( mode )
{
case 8:
input.raw = regs->rcx;
input_params_gpa = regs->rdx;
output_params_gpa = regs->r8;
break;
case 4:
input.raw = (regs->rdx << 32) | regs->_eax;
input_params_gpa = (regs->rbx << 32) | regs->_ecx;
output_params_gpa = (regs->rdi << 32) | regs->_esi;
break;
default:
goto out;
}
switch ( input.call_code )
{
case HvNotifyLongSpinWait:
/*
* See Microsoft Hypervisor Top Level Spec. section 18.5.1.
*/
perfc_incr(mshv_call_long_wait);
do_sched_op(SCHEDOP_yield, guest_handle_from_ptr(NULL, void));
status = HV_STATUS_SUCCESS;
break;
case HvFlushVirtualAddressSpace:
case HvFlushVirtualAddressList:
{
cpumask_t *pcpu_mask;
struct vcpu *v;
struct {
uint64_t address_space;
uint64_t flags;
uint64_t vcpu_mask;
} input_params;
/*
* See Microsoft Hypervisor Top Level Spec. sections 12.4.2
* and 12.4.3.
*/
perfc_incr(mshv_call_flush);
/* These hypercalls should never use the fast-call convention. */
status = HV_STATUS_INVALID_PARAMETER;
if ( input.fast )
break;
/* Get input parameters. */
if ( hvm_copy_from_guest_phys(&input_params, input_params_gpa,
sizeof(input_params)) != HVMCOPY_okay )
break;
/*
* It is not clear from the spec. if we are supposed to
* include current virtual CPU in the set or not in this case,
* so err on the safe side.
*/
if ( input_params.flags & HV_FLUSH_ALL_PROCESSORS )
input_params.vcpu_mask = ~0ul;
pcpu_mask = &this_cpu(ipi_cpumask);
cpumask_clear(pcpu_mask);
/*
* For each specified virtual CPU flush all ASIDs to invalidate
* TLB entries the next time it is scheduled and then, if it
* is currently running, add its physical CPU to a mask of
* those which need to be interrupted to force a flush.
*/
for_each_vcpu ( currd, v )
{
if ( v->vcpu_id >= (sizeof(input_params.vcpu_mask) * 8) )
break;
if ( !(input_params.vcpu_mask & (1ul << v->vcpu_id)) )
continue;
hvm_asid_flush_vcpu(v);
if ( v != curr && v->is_running )
__cpumask_set_cpu(v->processor, pcpu_mask);
}
/*
* Since ASIDs have now been flushed it just remains to
* force any CPUs currently running target vCPUs out of non-
* root mode. It's possible that re-scheduling has taken place
* so we may unnecessarily IPI some CPUs.
*/
if ( !cpumask_empty(pcpu_mask) )
smp_send_event_check_mask(pcpu_mask);
output.rep_complete = input.rep_count;
status = HV_STATUS_SUCCESS;
break;
}
default:
status = HV_STATUS_INVALID_HYPERCALL_CODE;
break;
}
out:
output.result = status;
switch (mode) {
case 8:
regs->rax = output.raw;
break;
default:
regs->rdx = output.raw >> 32;
regs->rax = (uint32_t)output.raw;
break;
}
return HVM_HCALL_completed;
}
int rdmsr_viridian_regs(uint32_t idx, uint64_t *val)
{
struct vcpu *v = current;
struct domain *d = v->domain;
if ( !is_viridian_domain(d) )
return 0;
switch ( idx )
{
case VIRIDIAN_MSR_GUEST_OS_ID:
perfc_incr(mshv_rdmsr_osid);
*val = d->arch.hvm_domain.viridian.guest_os_id.raw;
break;
case VIRIDIAN_MSR_HYPERCALL:
perfc_incr(mshv_rdmsr_hc_page);
*val = d->arch.hvm_domain.viridian.hypercall_gpa.raw;
break;
case VIRIDIAN_MSR_VP_INDEX:
perfc_incr(mshv_rdmsr_vp_index);
*val = v->vcpu_id;
break;
case VIRIDIAN_MSR_TSC_FREQUENCY:
if ( viridian_feature_mask(d) & HVMPV_no_freq )
return 0;
perfc_incr(mshv_rdmsr_tsc_frequency);
*val = (uint64_t)d->arch.tsc_khz * 1000ull;
break;
case VIRIDIAN_MSR_APIC_FREQUENCY:
if ( viridian_feature_mask(d) & HVMPV_no_freq )
return 0;
perfc_incr(mshv_rdmsr_apic_frequency);
*val = 1000000000ull / APIC_BUS_CYCLE_NS;
break;
case VIRIDIAN_MSR_ICR:
perfc_incr(mshv_rdmsr_icr);
*val = (((uint64_t)vlapic_get_reg(vcpu_vlapic(v), APIC_ICR2) << 32) |
vlapic_get_reg(vcpu_vlapic(v), APIC_ICR));
break;
case VIRIDIAN_MSR_TPR:
perfc_incr(mshv_rdmsr_tpr);
*val = vlapic_get_reg(vcpu_vlapic(v), APIC_TASKPRI);
break;
case VIRIDIAN_MSR_APIC_ASSIST:
perfc_incr(mshv_rdmsr_apic_msr);
*val = v->arch.hvm_vcpu.viridian.apic_assist.msr.raw;
break;
case VIRIDIAN_MSR_REFERENCE_TSC:
if ( !(viridian_feature_mask(d) & HVMPV_reference_tsc) )
return 0;
perfc_incr(mshv_rdmsr_tsc_msr);
*val = d->arch.hvm_domain.viridian.reference_tsc.raw;
break;
case VIRIDIAN_MSR_TIME_REF_COUNT:
{
struct viridian_time_ref_count *trc;
trc = &d->arch.hvm_domain.viridian.time_ref_count;
if ( !(viridian_feature_mask(d) & HVMPV_time_ref_count) )
return 0;
if ( !test_and_set_bit(_TRC_accessed, &trc->flags) )
printk(XENLOG_G_INFO "d%d: VIRIDIAN MSR_TIME_REF_COUNT: accessed\n",
d->domain_id);
perfc_incr(mshv_rdmsr_time_ref_count);
*val = raw_trc_val(d) + trc->off;
break;
}
default:
return 0;
}
return 1;
}
int wrmsr_viridian_regs(uint32_t idx, uint64_t val)
{
struct vcpu *v = current;
struct domain *d = v->domain;
if ( !is_viridian_domain(d) )
return 0;
switch ( idx )
{
case VIRIDIAN_MSR_GUEST_OS_ID:
perfc_incr(mshv_wrmsr_osid);
d->arch.hvm_domain.viridian.guest_os_id.raw = val;
dump_guest_os_id(d);
break;
case VIRIDIAN_MSR_HYPERCALL:
perfc_incr(mshv_wrmsr_hc_page);
d->arch.hvm_domain.viridian.hypercall_gpa.raw = val;
dump_hypercall(d);
if ( d->arch.hvm_domain.viridian.hypercall_gpa.fields.enabled )
enable_hypercall_page(d);
break;
case VIRIDIAN_MSR_VP_INDEX:
perfc_incr(mshv_wrmsr_vp_index);
break;
case VIRIDIAN_MSR_EOI:
perfc_incr(mshv_wrmsr_eoi);
vlapic_EOI_set(vcpu_vlapic(v));
break;
case VIRIDIAN_MSR_ICR: {
u32 eax = (u32)val, edx = (u32)(val >> 32);
struct vlapic *vlapic = vcpu_vlapic(v);
perfc_incr(mshv_wrmsr_icr);
eax &= ~(1 << 12);
edx &= 0xff000000;
vlapic_set_reg(vlapic, APIC_ICR2, edx);
vlapic_ipi(vlapic, eax, edx);
vlapic_set_reg(vlapic, APIC_ICR, eax);
break;
}
case VIRIDIAN_MSR_TPR:
perfc_incr(mshv_wrmsr_tpr);
vlapic_set_reg(vcpu_vlapic(v), APIC_TASKPRI, (uint8_t)val);
break;
case VIRIDIAN_MSR_APIC_ASSIST:
perfc_incr(mshv_wrmsr_apic_msr);
teardown_apic_assist(v); /* release any previous mapping */
v->arch.hvm_vcpu.viridian.apic_assist.msr.raw = val;
dump_apic_assist(v);
if ( v->arch.hvm_vcpu.viridian.apic_assist.msr.fields.enabled )
initialize_apic_assist(v);
break;
case VIRIDIAN_MSR_REFERENCE_TSC:
if ( !(viridian_feature_mask(d) & HVMPV_reference_tsc) )
return 0;
perfc_incr(mshv_wrmsr_tsc_msr);
d->arch.hvm_domain.viridian.reference_tsc.raw = val;
dump_reference_tsc(d);
if ( d->arch.hvm_domain.viridian.reference_tsc.fields.enabled )
update_reference_tsc(d, 1);
break;
default:
return 0;
}
return 1;
}
void event_check_interrupt(struct cpu_user_regs *regs)
{
ack_APIC_irq();
perfc_incr(ipis);
this_cpu(irq_count)++;
}
int viridian_hypercall(struct cpu_user_regs *regs)
{
int mode = hvm_guest_x86_mode(current);
unsigned long __attribute__((__unused__)) input_params_gpa, output_params_gpa;
uint16_t status = HV_STATUS_SUCCESS;
union hypercall_input {
uint64_t raw;
struct {
uint16_t call_code;
uint16_t rsvd1;
unsigned rep_count:12;
unsigned rsvd2:4;
unsigned rep_start:12;
unsigned rsvd3:4;
};
} input;
union hypercall_output {
uint64_t raw;
struct {
uint16_t result;
uint16_t rsvd1;
unsigned rep_complete:12;
unsigned rsvd2:20;
};
} output = { 0 };
ASSERT(is_viridian_domain(current->domain));
switch ( mode )
{
#ifdef __x86_64__
case 8:
input.raw = regs->rcx;
input_params_gpa = regs->rdx;
output_params_gpa = regs->r8;
break;
#endif
case 4:
input.raw = ((uint64_t)regs->edx << 32) | regs->eax;
input_params_gpa = ((uint64_t)regs->ebx << 32) | regs->ecx;
output_params_gpa = ((uint64_t)regs->edi << 32) | regs->esi;
break;
default:
goto out;
}
switch ( input.call_code )
{
case HvNotifyLongSpinWait:
perfc_incr(mshv_call_long_wait);
do_sched_op_compat(SCHEDOP_yield, 0);
status = HV_STATUS_SUCCESS;
break;
default:
status = HV_STATUS_INVALID_HYPERCALL_CODE;
break;
}
out:
output.result = status;
switch (mode) {
#ifdef __x86_64__
case 8:
regs->rax = output.raw;
break;
#endif
default:
regs->edx = output.raw >> 32;
regs->eax = output.raw;
break;
}
return HVM_HCALL_completed;
}
void apic_timer_interrupt(struct cpu_user_regs * regs)
{
ack_APIC_irq();
perfc_incr(apic_timer);
raise_softirq(TIMER_SOFTIRQ);
}
int wrmsr_viridian_regs(uint32_t idx, uint64_t val)
{
struct domain *d = current->domain;
if ( !is_viridian_domain(d) )
return 0;
switch ( idx )
{
case VIRIDIAN_MSR_GUEST_OS_ID:
perfc_incr(mshv_wrmsr_osid);
d->arch.hvm_domain.viridian.guest_os_id.raw = val;
gdprintk(XENLOG_INFO, "Guest os:\n");
gdprintk(XENLOG_INFO, "\tvendor: %x\n",
d->arch.hvm_domain.viridian.guest_os_id.fields.vendor);
gdprintk(XENLOG_INFO, "\tos: %x\n",
d->arch.hvm_domain.viridian.guest_os_id.fields.os);
gdprintk(XENLOG_INFO, "\tmajor: %x\n",
d->arch.hvm_domain.viridian.guest_os_id.fields.major);
gdprintk(XENLOG_INFO, "\tminor: %x\n",
d->arch.hvm_domain.viridian.guest_os_id.fields.minor);
gdprintk(XENLOG_INFO, "\tsp: %x\n",
d->arch.hvm_domain.viridian.guest_os_id.fields.service_pack);
gdprintk(XENLOG_INFO, "\tbuild: %x\n",
d->arch.hvm_domain.viridian.guest_os_id.fields.build_number);
break;
case VIRIDIAN_MSR_HYPERCALL:
perfc_incr(mshv_wrmsr_hc_page);
gdprintk(XENLOG_INFO, "Set hypercall page %"PRIx64".\n", val);
if ( d->arch.hvm_domain.viridian.guest_os_id.raw == 0 )
break;
d->arch.hvm_domain.viridian.hypercall_gpa.raw = val;
if ( d->arch.hvm_domain.viridian.hypercall_gpa.fields.enabled )
enable_hypercall_page();
break;
case VIRIDIAN_MSR_VP_INDEX:
perfc_incr(mshv_wrmsr_vp_index);
gdprintk(XENLOG_INFO, "Set VP index %"PRIu64".\n", val);
break;
case VIRIDIAN_MSR_EOI:
perfc_incr(mshv_wrmsr_eoi);
vlapic_EOI_set(vcpu_vlapic(current));
break;
case VIRIDIAN_MSR_ICR: {
u32 eax = (u32)val, edx = (u32)(val >> 32);
struct vlapic *vlapic = vcpu_vlapic(current);
perfc_incr(mshv_wrmsr_icr);
eax &= ~(1 << 12);
edx &= 0xff000000;
vlapic_set_reg(vlapic, APIC_ICR2, edx);
if ( vlapic_ipi(vlapic, eax, edx) == X86EMUL_OKAY )
vlapic_set_reg(vlapic, APIC_ICR, eax);
break;
}
case VIRIDIAN_MSR_TPR:
perfc_incr(mshv_wrmsr_tpr);
vlapic_set_reg(vcpu_vlapic(current), APIC_TASKPRI, (uint8_t)val);
break;
case VIRIDIAN_MSR_APIC_ASSIST:
/*
* We don't support the APIC assist page, and that fact is reflected in
* our CPUID flags. However, Windows 7 build 7000 has a bug which means
* that it doesn't recognise that, and tries to use the page anyway. We
* therefore have to fake up just enough to keep win7 happy.
* Fortunately, that's really easy: just setting the first four bytes
* in the page to zero effectively disables the page again, so that's
* what we do. Semantically, the first four bytes are supposed to be a
* flag saying whether the guest really needs to issue an EOI. Setting
* that flag to zero means that it must always issue one, which is what
* we want. Once a page has been repurposed as an APIC assist page the
* guest isn't allowed to set anything in it, so the flag remains zero
* and all is fine. The guest is allowed to clear flags in the page,
* but that doesn't cause us any problems.
*/
if ( val & 1 ) /* APIC assist page enabled? */
{
uint32_t word = 0;
paddr_t page_start = val & ~1ul;
(void)hvm_copy_to_guest_phys(page_start, &word, sizeof(word));
}
break;
default:
return 0;
}
return 1;
}
