void vmm_scheduler_irq_exit(arch_regs_t *regs)
{
struct vmm_scheduler_ctrl *schedp = &this_cpu(sched);
struct vmm_vcpu *vcpu = NULL;
/* Determine current vcpu */
vcpu = schedp->current_vcpu;
if (!vcpu) {
return;
}
/* If current vcpu is not RUNNING or yield on exit is set
* then context switch
*/
if ((vmm_manager_vcpu_get_state(vcpu) != VMM_VCPU_STATE_RUNNING) ||
schedp->yield_on_irq_exit) {
vmm_scheduler_next(schedp, &schedp->ev, schedp->irq_regs);
schedp->yield_on_irq_exit = FALSE;
}
/* VCPU irq processing */
vmm_vcpu_irq_process(vcpu, regs);
/* Indicate that we have exited IRQ */
schedp->irq_context = FALSE;
/* Clear pointer to IRQ registers */
schedp->irq_regs = NULL;
}
void vmm_scheduler_yield(void)
{
irq_flags_t flags;
struct vmm_scheduler_ctrl *schedp = &this_cpu(sched);
arch_cpu_irq_save(flags);
if (schedp->irq_context) {
vmm_panic("%s: Cannot yield in IRQ context\n", __func__);
}
if (!schedp->current_vcpu) {
vmm_panic("%s: NULL VCPU pointer\n", __func__);
}
if (schedp->current_vcpu->is_normal) {
/* For Normal VCPU
* Just enable yield on exit and rest will be taken care
* by vmm_scheduler_irq_exit()
*/
if (vmm_manager_vcpu_get_state(schedp->current_vcpu) ==
VMM_VCPU_STATE_RUNNING) {
schedp->yield_on_irq_exit = TRUE;
}
} else {
/* For Orphan VCPU
* Forcefully expire yield
*/
arch_vcpu_preempt_orphan();
}
arch_cpu_irq_restore(flags);
}
void vmm_vcpu_irq_assert(struct vmm_vcpu *vcpu, u32 irq_no, u64 reason)
{
/* For non-normal VCPU dont do anything */
if (!vcpu || !vcpu->is_normal) {
return;
}
/* If VCPU is not in interruptible state then dont do anything */
if (!(vmm_manager_vcpu_get_state(vcpu) & VMM_VCPU_STATE_INTERRUPTIBLE)) {
return;
}
/* Check irq number */
if (irq_no > vcpu->irqs.irq_count) {
return;
}
/* Assert the irq */
if (arch_atomic_cmpxchg(&vcpu->irqs.irq[irq_no].assert,
DEASSERTED, ASSERTED) == DEASSERTED) {
if (arch_vcpu_irq_assert(vcpu, irq_no, reason) == VMM_OK) {
vcpu->irqs.irq[irq_no].reason = reason;
arch_atomic_inc(&vcpu->irqs.execute_pending);
arch_atomic64_inc(&vcpu->irqs.assert_count);
} else {
arch_atomic_write(&vcpu->irqs.irq[irq_no].assert,
DEASSERTED);
}
}
/* Resume VCPU from wfi */
vcpu_irq_wfi_resume(vcpu, FALSE);
}
void vmm_vcpu_irq_process(struct vmm_vcpu *vcpu, arch_regs_t *regs)
{
/* For non-normal vcpu dont do anything */
if (!vcpu || !vcpu->is_normal) {
return;
}
/* If vcpu is not in interruptible state then dont do anything */
if (!(vmm_manager_vcpu_get_state(vcpu) &
VMM_VCPU_STATE_INTERRUPTIBLE)) {
return;
}
/* Proceed only if we have pending execute */
if (arch_atomic_dec_if_positive(&vcpu->irqs.execute_pending) >= 0) {
int irq_no = -1;
u32 i, tmp_prio, irq_count = vcpu->irqs.irq_count;
u32 irq_prio = 0;
/* Find the irq number to process */
for (i = 0; i < irq_count; i++) {
if (arch_atomic_read(&vcpu->irqs.irq[i].assert) ==
ASSERTED) {
tmp_prio = arch_vcpu_irq_priority(vcpu, i);
if (tmp_prio > irq_prio) {
irq_no = i;
irq_prio = tmp_prio;
}
}
}
if (irq_no == -1) {
return;
}
/* If irq number found then execute it */
if (arch_atomic_cmpxchg(&vcpu->irqs.irq[irq_no].assert,
ASSERTED, PENDING) == ASSERTED) {
if (arch_vcpu_irq_execute(vcpu, regs, irq_no,
vcpu->irqs.irq[irq_no].reason) == VMM_OK) {
arch_atomic_write(&vcpu->irqs.
irq[irq_no].assert,
DEASSERTED);
arch_atomic64_inc(&vcpu->irqs.
execute_count);
} else {
/* arch_vcpu_irq_execute failed may be
* because VCPU was already processing
* a VCPU irq hence increment execute
* pending count to try next time.
*/
arch_atomic_inc(&vcpu->irqs.
execute_pending);
arch_atomic_write(&vcpu->irqs.
irq[irq_no].assert,
ASSERTED);
}
}
}
}
void cpu_vcpu_halt(struct vmm_vcpu *vcpu, arch_regs_t *regs)
{
if (vmm_manager_vcpu_get_state(vcpu) != VMM_VCPU_STATE_HALTED) {
vmm_printf("\n");
cpu_vcpu_dump_user_reg(regs);
vmm_manager_vcpu_halt(vcpu);
}
}
void do_sync(arch_regs_t *regs, unsigned long mode)
{
int rc = VMM_OK;
u32 ec, il, iss;
u64 esr, far, elr;
physical_addr_t fipa = 0;
struct vmm_vcpu *vcpu;
esr = mrs(esr_el2);
far = mrs(far_el2);
elr = mrs(elr_el2);
ec = (esr & ESR_EC_MASK) >> ESR_EC_SHIFT;
il = (esr & ESR_IL_MASK) >> ESR_IL_SHIFT;
iss = (esr & ESR_ISS_MASK) >> ESR_ISS_SHIFT;
vcpu = vmm_scheduler_current_vcpu();
/* We dont expect any faults from hypervisor code itself
* so, any trap we get from hypervisor mode means something
* unexpected has occured.
*/
if ((regs->pstate & PSR_EL_MASK) == PSR_EL_2) {
if ((ec == EC_TRAP_HVC_A64) && (iss == 0)) {
vmm_scheduler_preempt_orphan(regs);
return;
}
vmm_printf("%s: CPU%d VCPU=%s unexpected exception\n",
__func__, vmm_smp_processor_id(),
(vcpu) ? vcpu->name : "(NULL)");
vmm_printf("%s: ESR=0x%016lx EC=0x%x IL=0x%x ISS=0x%x\n",
__func__, esr, ec, il, iss);
vmm_printf("%s: ELR=0x%016lx FAR=0x%016lx HPFAR=0x%016lx\n",
__func__, elr, far, mrs(hpfar_el2));
cpu_vcpu_dump_user_reg(regs);
vmm_panic("%s: please reboot ...\n", __func__);
}
vmm_scheduler_irq_enter(regs, TRUE);
switch (ec) {
case EC_UNKNOWN:
/* We dont expect to get this trap so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_WFI_WFE:
/* WFI emulation */
rc = cpu_vcpu_emulate_wfi_wfe(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP15_A32:
/* MCR/MRC CP15 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCRR_MRRC_CP15_A32:
/* MCRR/MRRC CP15 emulation */
rc = cpu_vcpu_emulate_mcrr_mrrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP14_A32:
/* MCR/MRC CP14 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_LDC_STC_CP14_A32:
/* LDC/STC CP14 emulation */
rc = cpu_vcpu_emulate_ldc_stc_cp14(vcpu, regs, il, iss);
break;
case EC_SIMD_FPU:
/* Advanced SIMD and FPU emulation */
rc = cpu_vcpu_emulate_simd_fp_regs(vcpu, regs, il, iss);
break;
case EC_FPEXC_A32:
case EC_FPEXC_A64:
/* We dont expect any FP execution faults */
rc = VMM_EFAIL;
break;
case EC_TRAP_MRC_VMRS_CP10_A32:
/* MRC (or VMRS) to CP10 for MVFR0, MVFR1 or FPSID */
rc = cpu_vcpu_emulate_vmrs(vcpu, regs, il, iss);
break;
case EC_TRAP_MCRR_MRRC_CP14_A32:
/* MRRC to CP14 emulation */
rc = cpu_vcpu_emulate_mcrr_mrrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_SVC_A32:
case EC_TRAP_SVC_A64:
/* We dont expect to get these traps so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_SMC_A32:
/* SMC emulation for A32 guest */
rc = cpu_vcpu_emulate_smc32(vcpu, regs, il, iss);
break;
case EC_TRAP_SMC_A64:
/* SMC emulation for A64 guest */
rc = cpu_vcpu_emulate_smc64(vcpu, regs, il, iss);
break;
case EC_TRAP_HVC_A32:
/* HVC emulation for A32 guest */
rc = cpu_vcpu_emulate_hvc32(vcpu, regs, il, iss);
break;
case EC_TRAP_HVC_A64:
/* HVC emulation for A64 guest */
rc = cpu_vcpu_emulate_hvc64(vcpu, regs, il, iss);
break;
case EC_TRAP_MSR_MRS_SYSTEM:
/* MSR/MRS/SystemRegs emulation */
rc = cpu_vcpu_emulate_msr_mrs_system(vcpu, regs, il, iss);
break;
case EC_TRAP_LWREL_INST_ABORT:
/* Stage2 instruction abort */
fipa = (mrs(hpfar_el2) & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (mrs(far_el2) & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_inst_abort(vcpu, regs, il, iss, fipa);
break;
case EC_TRAP_LWREL_DATA_ABORT:
/* Stage2 data abort */
fipa = (mrs(hpfar_el2) & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (mrs(far_el2) & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_data_abort(vcpu, regs, il, iss, fipa);
break;
case EC_CUREL_INST_ABORT:
case EC_CUREL_DATA_ABORT:
case EC_SERROR:
/* We dont expect to get aborts from EL2 so error */
rc = VMM_EFAIL;
break;
case EC_PC_UNALIGNED:
case EC_SP_UNALIGNED:
/* We dont expect to get alignment faults from EL2 */
rc = VMM_EFAIL;
break;
default:
/* Unhandled or unknown EC value so error */
rc = VMM_EFAIL;
break;
};
if (rc) {
vmm_printf("%s: CPU%d VCPU=%s sync failed (error %d)\n",
__func__, vmm_smp_processor_id(),
(vcpu) ? vcpu->name : "(NULL)", rc);
vmm_printf("%s: ESR=0x%016lx EC=0x%x IL=0x%x ISS=0x%x\n",
__func__, esr, ec, il, iss);
vmm_printf("%s: ELR=0x%016lx FAR=0x%016lx HPFAR=0x%016lx\n",
__func__, elr, far, mrs(hpfar_el2));
if (vmm_manager_vcpu_get_state(vcpu) != VMM_VCPU_STATE_HALTED) {
cpu_vcpu_halt(vcpu, regs);
}
}
vmm_scheduler_irq_exit(regs);
}
void do_hyp_trap(arch_regs_t *regs)
{
int rc = VMM_OK;
u32 hsr, ec, il, iss;
virtual_addr_t far;
physical_addr_t fipa = 0;
struct vmm_vcpu *vcpu;
hsr = read_hsr();
ec = (hsr & HSR_EC_MASK) >> HSR_EC_SHIFT;
il = (hsr & HSR_IL_MASK) >> HSR_IL_SHIFT;
iss = (hsr & HSR_ISS_MASK) >> HSR_ISS_SHIFT;
vcpu = vmm_scheduler_current_vcpu();
/* We dont expect any faults from hypervisor code itself
* so, any trap we get from hypervisor mode means something
* unexpected has occured.
*/
if ((regs->cpsr & CPSR_MODE_MASK) == CPSR_MODE_HYPERVISOR) {
vmm_printf("%s: CPU%d unexpected exception\n",
__func__, vmm_smp_processor_id());
vmm_printf("%s: Current VCPU=%s HSR=0x%08x\n",
__func__, (vcpu) ? vcpu->name : "(NULL)",
read_hsr());
vmm_printf("%s: HPFAR=0x%08x HIFAR=0x%08x HDFAR=0x%08x\n",
__func__, read_hpfar(), read_hifar(), read_hdfar());
cpu_vcpu_dump_user_reg(regs);
vmm_panic("%s: please reboot ...\n", __func__);
}
vmm_scheduler_irq_enter(regs, TRUE);
switch (ec) {
case EC_UNKNOWN:
/* We dont expect to get this trap so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_WFI_WFE:
/* WFI emulation */
rc = cpu_vcpu_emulate_wfi_wfe(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP15:
/* MCR/MRC CP15 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCRR_MRRC_CP15:
/* MCRR/MRRC CP15 emulation */
rc = cpu_vcpu_emulate_mcrr_mrrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP14:
/* MCR/MRC CP14 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_LDC_STC_CP14:
/* LDC/STC CP14 emulation */
rc = cpu_vcpu_emulate_ldc_stc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_CP0_TO_CP13:
/* CP0 to CP13 emulation */
rc = cpu_vcpu_emulate_cp0_cp13(vcpu, regs, il, iss);
break;
case EC_TRAP_VMRS:
/* MRC (or VMRS) to CP10 for MVFR0, MVFR1 or FPSID */
rc = cpu_vcpu_emulate_vmrs(vcpu, regs, il, iss);
break;
case EC_TRAP_JAZELLE:
/* Jazelle emulation */
rc = cpu_vcpu_emulate_jazelle(vcpu, regs, il, iss);
break;
case EC_TRAP_BXJ:
/* BXJ emulation */
rc = cpu_vcpu_emulate_bxj(vcpu, regs, il, iss);
break;
case EC_TRAP_MRRC_CP14:
/* MRRC to CP14 emulation */
rc = cpu_vcpu_emulate_mrrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_SVC:
/* We dont expect to get this trap so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_HVC:
/* Hypercall or HVC emulation */
rc = cpu_vcpu_emulate_hvc(vcpu, regs, il, iss);
break;
case EC_TRAP_SMC:
/* System Monitor Call or SMC emulation */
rc = cpu_vcpu_emulate_smc(vcpu, regs, il, iss);
break;
case EC_TRAP_STAGE2_INST_ABORT:
/* Stage2 instruction abort */
far = read_hifar();
fipa = (read_hpfar() & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (far & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_inst_abort(vcpu, regs, il, iss, far, fipa);
break;
case EC_TRAP_STAGE1_INST_ABORT:
/* We dont expect to get this trap so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_STAGE2_DATA_ABORT:
/* Stage2 data abort */
far = read_hdfar();
fipa = (read_hpfar() & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (far & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_data_abort(vcpu, regs, il, iss, far, fipa);
break;
case EC_TRAP_STAGE1_DATA_ABORT:
/* We dont expect to get this trap so error */
rc = VMM_EFAIL;
break;
default:
/* Unknown EC value so error */
rc = VMM_EFAIL;
break;
};
if (rc) {
vmm_printf("\n%s: ec=0x%x, il=0x%x, iss=0x%x,"
" fipa=0x%x, error=%d\n", __func__,
ec, il, iss, fipa, rc);
if (vmm_manager_vcpu_get_state(vcpu) != VMM_VCPU_STATE_HALTED) {
cpu_vcpu_halt(vcpu, regs);
}
}
vmm_scheduler_irq_exit(regs);
}
