void do_handle_sync(arch_regs_t *regs,
unsigned long exc, unsigned long baddr)
{
if ((exc == EXC_STORE_AMO_PAGE_FAULT) &&
(regs->sstatus & SR_SPP) &&
(regs->sepc == preempt_orphan_pc)) {
regs->sepc += 4;
vmm_scheduler_preempt_orphan(regs);
return;
}
vmm_scheduler_irq_enter(regs, TRUE);
/* TODO: */
vmm_scheduler_irq_exit(regs);
}
void do_soft_irq(arch_regs_t *regs)
{
struct vmm_vcpu *vcpu;
if ((regs->cpsr & CPSR_MODE_MASK) == CPSR_MODE_HYPERVISOR) {
vmm_scheduler_preempt_orphan(regs);
return;
} else {
vcpu = vmm_scheduler_current_vcpu();
vmm_printf("%s: CPU%d unexpected exception\n",
__func__, vmm_smp_processor_id());
vmm_printf("%s: Current VCPU=%s HSR=0x%08x\n",
__func__, (vcpu) ? vcpu->name : "(NULL)",
read_hsr());
vmm_printf("%s: HPFAR=0x%08x HIFAR=0x%08x HDFAR=0x%08x\n",
__func__, read_hpfar(), read_hifar(), read_hdfar());
cpu_vcpu_dump_user_reg(regs);
vmm_panic("%s: please reboot ...\n", __func__);
}
}
int vmm_scheduler_state_change(struct vmm_vcpu *vcpu, u32 new_state)
{
u64 tstamp;
int rc = VMM_OK;
irq_flags_t flags;
bool preempt = FALSE;
u32 chcpu = vmm_smp_processor_id(), vhcpu;
struct vmm_scheduler_ctrl *schedp;
u32 current_state;
if (!vcpu) {
return VMM_EFAIL;
}
vmm_write_lock_irqsave_lite(&vcpu->sched_lock, flags);
vhcpu = vcpu->hcpu;
schedp = &per_cpu(sched, vhcpu);
current_state = arch_atomic_read(&vcpu->state);
switch(new_state) {
case VMM_VCPU_STATE_UNKNOWN:
/* Existing VCPU being destroyed */
rc = vmm_schedalgo_vcpu_cleanup(vcpu);
break;
case VMM_VCPU_STATE_RESET:
if (current_state == VMM_VCPU_STATE_UNKNOWN) {
/* New VCPU */
rc = vmm_schedalgo_vcpu_setup(vcpu);
} else if (current_state != VMM_VCPU_STATE_RESET) {
/* Existing VCPU */
/* Make sure VCPU is not in a ready queue */
if ((schedp->current_vcpu != vcpu) &&
(current_state == VMM_VCPU_STATE_READY)) {
if ((rc = rq_detach(schedp, vcpu))) {
break;
}
}
/* Make sure current VCPU is preempted */
if ((schedp->current_vcpu == vcpu) &&
(current_state == VMM_VCPU_STATE_RUNNING)) {
preempt = TRUE;
}
vcpu->reset_count++;
if ((rc = arch_vcpu_init(vcpu))) {
break;
}
if ((rc = vmm_vcpu_irq_init(vcpu))) {
break;
}
} else {
rc = VMM_EFAIL;
}
break;
case VMM_VCPU_STATE_READY:
if ((current_state == VMM_VCPU_STATE_RESET) ||
(current_state == VMM_VCPU_STATE_PAUSED)) {
/* Enqueue VCPU to ready queue */
rc = rq_enqueue(schedp, vcpu);
if (!rc && (schedp->current_vcpu != vcpu)) {
preempt = rq_prempt_needed(schedp);
}
} else {
rc = VMM_EFAIL;
}
break;
case VMM_VCPU_STATE_PAUSED:
case VMM_VCPU_STATE_HALTED:
if ((current_state == VMM_VCPU_STATE_READY) ||
(current_state == VMM_VCPU_STATE_RUNNING)) {
/* Expire timer event if current VCPU
* is paused or halted
*/
if (schedp->current_vcpu == vcpu) {
preempt = TRUE;
} else if (current_state == VMM_VCPU_STATE_READY) {
/* Make sure VCPU is not in a ready queue */
rc = rq_detach(schedp, vcpu);
}
} else {
rc = VMM_EFAIL;
}
break;
}
if (rc == VMM_OK) {
tstamp = vmm_timer_timestamp();
switch (current_state) {
case VMM_VCPU_STATE_READY:
vcpu->state_ready_nsecs +=
tstamp - vcpu->state_tstamp;
break;
case VMM_VCPU_STATE_RUNNING:
vcpu->state_running_nsecs +=
tstamp - vcpu->state_tstamp;
break;
case VMM_VCPU_STATE_PAUSED:
vcpu->state_paused_nsecs +=
tstamp - vcpu->state_tstamp;
break;
case VMM_VCPU_STATE_HALTED:
vcpu->state_halted_nsecs +=
tstamp - vcpu->state_tstamp;
break;
default:
break;
}
if (new_state == VMM_VCPU_STATE_RESET) {
vcpu->state_ready_nsecs = 0;
vcpu->state_running_nsecs = 0;
vcpu->state_paused_nsecs = 0;
vcpu->state_halted_nsecs = 0;
vcpu->reset_tstamp = tstamp;
}
arch_atomic_write(&vcpu->state, new_state);
vcpu->state_tstamp = tstamp;
}
vmm_write_unlock_irqrestore_lite(&vcpu->sched_lock, flags);
if (preempt && schedp->current_vcpu) {
if (chcpu == vhcpu) {
if (schedp->current_vcpu->is_normal) {
schedp->yield_on_irq_exit = TRUE;
} else if (schedp->irq_context) {
vmm_scheduler_preempt_orphan(schedp->irq_regs);
} else {
arch_vcpu_preempt_orphan();
}
} else {
vmm_smp_ipi_async_call(vmm_cpumask_of(vhcpu),
scheduler_ipi_resched,
NULL, NULL, NULL);
}
}
return rc;
}
void do_sync(arch_regs_t *regs, unsigned long mode)
{
int rc = VMM_OK;
u32 ec, il, iss;
u64 esr, far, elr;
physical_addr_t fipa = 0;
struct vmm_vcpu *vcpu;
esr = mrs(esr_el2);
far = mrs(far_el2);
elr = mrs(elr_el2);
ec = (esr & ESR_EC_MASK) >> ESR_EC_SHIFT;
il = (esr & ESR_IL_MASK) >> ESR_IL_SHIFT;
iss = (esr & ESR_ISS_MASK) >> ESR_ISS_SHIFT;
vcpu = vmm_scheduler_current_vcpu();
/* We dont expect any faults from hypervisor code itself
* so, any trap we get from hypervisor mode means something
* unexpected has occured.
*/
if ((regs->pstate & PSR_EL_MASK) == PSR_EL_2) {
if ((ec == EC_TRAP_HVC_A64) && (iss == 0)) {
vmm_scheduler_preempt_orphan(regs);
return;
}
vmm_printf("%s: CPU%d VCPU=%s unexpected exception\n",
__func__, vmm_smp_processor_id(),
(vcpu) ? vcpu->name : "(NULL)");
vmm_printf("%s: ESR=0x%016lx EC=0x%x IL=0x%x ISS=0x%x\n",
__func__, esr, ec, il, iss);
vmm_printf("%s: ELR=0x%016lx FAR=0x%016lx HPFAR=0x%016lx\n",
__func__, elr, far, mrs(hpfar_el2));
cpu_vcpu_dump_user_reg(regs);
vmm_panic("%s: please reboot ...\n", __func__);
}
vmm_scheduler_irq_enter(regs, TRUE);
switch (ec) {
case EC_UNKNOWN:
/* We dont expect to get this trap so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_WFI_WFE:
/* WFI emulation */
rc = cpu_vcpu_emulate_wfi_wfe(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP15_A32:
/* MCR/MRC CP15 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCRR_MRRC_CP15_A32:
/* MCRR/MRRC CP15 emulation */
rc = cpu_vcpu_emulate_mcrr_mrrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP14_A32:
/* MCR/MRC CP14 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_LDC_STC_CP14_A32:
/* LDC/STC CP14 emulation */
rc = cpu_vcpu_emulate_ldc_stc_cp14(vcpu, regs, il, iss);
break;
case EC_SIMD_FPU:
/* Advanced SIMD and FPU emulation */
rc = cpu_vcpu_emulate_simd_fp_regs(vcpu, regs, il, iss);
break;
case EC_FPEXC_A32:
case EC_FPEXC_A64:
/* We dont expect any FP execution faults */
rc = VMM_EFAIL;
break;
case EC_TRAP_MRC_VMRS_CP10_A32:
/* MRC (or VMRS) to CP10 for MVFR0, MVFR1 or FPSID */
rc = cpu_vcpu_emulate_vmrs(vcpu, regs, il, iss);
break;
case EC_TRAP_MCRR_MRRC_CP14_A32:
/* MRRC to CP14 emulation */
rc = cpu_vcpu_emulate_mcrr_mrrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_SVC_A32:
case EC_TRAP_SVC_A64:
/* We dont expect to get these traps so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_SMC_A32:
/* SMC emulation for A32 guest */
rc = cpu_vcpu_emulate_smc32(vcpu, regs, il, iss);
break;
case EC_TRAP_SMC_A64:
/* SMC emulation for A64 guest */
rc = cpu_vcpu_emulate_smc64(vcpu, regs, il, iss);
break;
case EC_TRAP_HVC_A32:
/* HVC emulation for A32 guest */
rc = cpu_vcpu_emulate_hvc32(vcpu, regs, il, iss);
break;
case EC_TRAP_HVC_A64:
/* HVC emulation for A64 guest */
rc = cpu_vcpu_emulate_hvc64(vcpu, regs, il, iss);
break;
case EC_TRAP_MSR_MRS_SYSTEM:
/* MSR/MRS/SystemRegs emulation */
rc = cpu_vcpu_emulate_msr_mrs_system(vcpu, regs, il, iss);
break;
case EC_TRAP_LWREL_INST_ABORT:
/* Stage2 instruction abort */
fipa = (mrs(hpfar_el2) & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (mrs(far_el2) & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_inst_abort(vcpu, regs, il, iss, fipa);
break;
case EC_TRAP_LWREL_DATA_ABORT:
/* Stage2 data abort */
fipa = (mrs(hpfar_el2) & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (mrs(far_el2) & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_data_abort(vcpu, regs, il, iss, fipa);
break;
case EC_CUREL_INST_ABORT:
case EC_CUREL_DATA_ABORT:
case EC_SERROR:
/* We dont expect to get aborts from EL2 so error */
rc = VMM_EFAIL;
break;
case EC_PC_UNALIGNED:
case EC_SP_UNALIGNED:
/* We dont expect to get alignment faults from EL2 */
rc = VMM_EFAIL;
break;
default:
/* Unhandled or unknown EC value so error */
rc = VMM_EFAIL;
break;
};
if (rc) {
vmm_printf("%s: CPU%d VCPU=%s sync failed (error %d)\n",
__func__, vmm_smp_processor_id(),
(vcpu) ? vcpu->name : "(NULL)", rc);
vmm_printf("%s: ESR=0x%016lx EC=0x%x IL=0x%x ISS=0x%x\n",
__func__, esr, ec, il, iss);
vmm_printf("%s: ELR=0x%016lx FAR=0x%016lx HPFAR=0x%016lx\n",
__func__, elr, far, mrs(hpfar_el2));
if (vmm_manager_vcpu_get_state(vcpu) != VMM_VCPU_STATE_HALTED) {
cpu_vcpu_halt(vcpu, regs);
}
}
vmm_scheduler_irq_exit(regs);
}
void do_handle_trap(arch_regs_t *regs, unsigned long cause)
{
int rc = VMM_OK;
bool panic = TRUE;
const char *msg = "trap handling failed";
struct vmm_vcpu *vcpu;
if ((cause == CAUSE_STORE_PAGE_FAULT) &&
!(regs->hstatus & HSTATUS_SPV) &&
(regs->sepc == preempt_orphan_pc)) {
regs->sepc += 4;
vmm_scheduler_preempt_orphan(regs);
return;
}
vmm_scheduler_irq_enter(regs, TRUE);
vcpu = vmm_scheduler_current_vcpu();
if (!vcpu || !vcpu->is_normal) {
rc = VMM_EFAIL;
msg = "unexpected trap";
goto done;
}
switch (cause) {
case CAUSE_ILLEGAL_INSTRUCTION:
msg = "illegal instruction fault failed";
if (regs->hstatus & HSTATUS_SPV) {
rc = cpu_vcpu_illegal_insn_fault(vcpu, regs,
csr_read(stval));
panic = FALSE;
} else {
rc = VMM_EINVALID;
}
break;
case CAUSE_FETCH_PAGE_FAULT:
case CAUSE_LOAD_PAGE_FAULT:
case CAUSE_STORE_PAGE_FAULT:
msg = "page fault failed";
if ((regs->hstatus & HSTATUS_SPV) &&
(regs->hstatus & HSTATUS_STL)) {
rc = cpu_vcpu_page_fault(vcpu, regs,
cause, csr_read(stval));
panic = FALSE;
} else {
rc = VMM_EINVALID;
}
break;
default:
rc = VMM_EFAIL;
break;
};
if (rc) {
vmm_manager_vcpu_halt(vcpu);
}
done:
if (rc) {
do_error(vcpu, regs, cause, msg, rc, panic);
}
vmm_scheduler_irq_exit(regs);
}