void do_bad_mode(arch_regs_t *regs, unsigned long mode)
{
u32 ec, il, iss;
u64 esr, far, elr;
struct vmm_vcpu *vcpu;
esr = mrs(esr_el2);
far = mrs(far_el2);
elr = mrs(elr_el2);
ec = (esr & ESR_EC_MASK) >> ESR_EC_SHIFT;
il = (esr & ESR_IL_MASK) >> ESR_IL_SHIFT;
iss = (esr & ESR_ISS_MASK) >> ESR_ISS_SHIFT;
vcpu = vmm_scheduler_current_vcpu();
vmm_printf("%s: CPU%d VCPU=%s unexpected exception\n",
__func__, vmm_smp_processor_id(),
(vcpu) ? vcpu->name : "(NULL)");
vmm_printf("%s: ESR=0x%016lx EC=0x%x IL=0x%x ISS=0x%x\n",
__func__, esr, ec, il, iss);
vmm_printf("%s: ELR=0x%016lx FAR=0x%016lx HPFAR=0x%016lx\n",
__func__, elr, far, mrs(hpfar_el2));
cpu_vcpu_dump_user_reg(regs);
vmm_panic("%s: please reboot ...\n", __func__);
}
void arm_sync_abort(struct pt_regs *regs)
{
int ite;
u64 esr, far, ec, iss;
esr = mrs(esr_el1);
far = mrs(far_el1);
ec = (esr & ESR_EC_MASK) >> ESR_EC_SHIFT;
iss = (esr & ESR_ISS_MASK) >> ESR_ISS_SHIFT;
basic_printf("Bad synchronous exception @ PC: 0x%lX\n",
(virtual_addr_t)regs->pc);
basic_printf("ESR: 0x%08X (EC:0x%X, ISS:0x%X)\n",
(u32)esr, (u32)ec, (u32)iss);
basic_printf("LR: 0x%lX, FAR: 0x%lX, PSTATE: 0x%X\n",
(virtual_addr_t)regs->lr, (virtual_addr_t)far,
(u32)regs->pstate);
basic_printf(" General Purpose Registers");
for (ite = 0; ite < 30; ite++) {
if (ite % 2 == 0)
basic_printf("\n");
basic_printf(" X%02d=0x%016lx ",
ite, (unsigned long)regs->gpr[ite]);
}
basic_printf("\n");
while(1);
}
void NesCpuTranslator::mSaveInternalFlags()
{
#if !defined(FRAME_POINTER_FOR_GDB)
__ mrs(mInternalFlagsCopy, CPSR);
#else
__ mrs(ip, CPSR);
__ str(ip, MemOperand(mDataBase, offsetof(NesCpuRecData,internalFlagsCopy)));
#endif
}
int arch_vcpu_deinit(struct vmm_vcpu *vcpu)
{
int rc = VMM_OK;
u32 saved_cptr_el2, saved_hstr_el2;
/* For both Orphan & Normal VCPUs */
memset(arm_regs(vcpu), 0, sizeof(arch_regs_t));
/* For Orphan VCPUs do nothing else */
if (!vcpu->is_normal) {
return VMM_OK;
}
/* Save CPTR_EL2 and HSTR_EL2 */
saved_cptr_el2 = mrs(cptr_el2);
saved_hstr_el2 = mrs(hstr_el2);
/* We force disable coprocessor and system traps to be
* consistent with arch_vcpu_init() function.
*/
msr(cptr_el2, 0x0);
msr(hstr_el2, 0x0);
/* Free Generic Timer Context */
if (arm_feature(vcpu, ARM_FEATURE_GENERIC_TIMER)) {
if ((rc = generic_timer_vcpu_context_deinit(vcpu,
&arm_gentimer_context(vcpu)))) {
goto done;
}
}
/* Free VFP context */
rc = cpu_vcpu_vfp_deinit(vcpu);
if (rc) {
goto done;
}
/* Free sysregs context */
rc = cpu_vcpu_sysregs_deinit(vcpu);
if (rc) {
goto done;
}
/* Free private context */
vmm_free(vcpu->arch_priv);
vcpu->arch_priv = NULL;
rc = VMM_OK;
done:
msr(cptr_el2, saved_cptr_el2);
msr(hstr_el2, saved_hstr_el2);
return VMM_OK;
}
void MacroAssembler::lowLevelDebug(const char *s,
Register ra,
Register rb,
Register rc)
{
Q_ASSERT(ra.code() < 13 && rb.code() < 13 && rc.code() < 13);
int preserved = 0x1fff | lr.bit();
stm(db_w, sp, preserved);
add(fp, sp, Operand(12*4));
mrs(r4, CPSR);
Label omitString;
b(&omitString);
int sPtr = intptr_t(buffer_ + pcOffset());
do {
db(*s);
} while (*(s++));
while (pcOffset() & 3)
db('\0');
bind(&omitString);
ldr(r3, MemOperand(sp, rc.code()*4));
ldr(r2, MemOperand(sp, rb.code()*4));
ldr(r1, MemOperand(sp, ra.code()*4));
mov(r0, Operand(sPtr));
void (*qDebugPtr)(const char *,...) = &qDebug;
mov(ip, Operand(intptr_t(qDebugPtr)));
blx(ip);
msr(CPSR_f, Operand(r4));
ldm(ia_w, sp, preserved);
}
void do_sync(arch_regs_t *regs, unsigned long mode)
{
int rc = VMM_OK;
u32 ec, il, iss;
u64 esr, far, elr;
physical_addr_t fipa = 0;
struct vmm_vcpu *vcpu;
esr = mrs(esr_el2);
far = mrs(far_el2);
elr = mrs(elr_el2);
ec = (esr & ESR_EC_MASK) >> ESR_EC_SHIFT;
il = (esr & ESR_IL_MASK) >> ESR_IL_SHIFT;
iss = (esr & ESR_ISS_MASK) >> ESR_ISS_SHIFT;
vcpu = vmm_scheduler_current_vcpu();
/* We dont expect any faults from hypervisor code itself
* so, any trap we get from hypervisor mode means something
* unexpected has occured.
*/
if ((regs->pstate & PSR_EL_MASK) == PSR_EL_2) {
if ((ec == EC_TRAP_HVC_A64) && (iss == 0)) {
vmm_scheduler_preempt_orphan(regs);
return;
}
vmm_printf("%s: CPU%d VCPU=%s unexpected exception\n",
__func__, vmm_smp_processor_id(),
(vcpu) ? vcpu->name : "(NULL)");
vmm_printf("%s: ESR=0x%016lx EC=0x%x IL=0x%x ISS=0x%x\n",
__func__, esr, ec, il, iss);
vmm_printf("%s: ELR=0x%016lx FAR=0x%016lx HPFAR=0x%016lx\n",
__func__, elr, far, mrs(hpfar_el2));
cpu_vcpu_dump_user_reg(regs);
vmm_panic("%s: please reboot ...\n", __func__);
}
vmm_scheduler_irq_enter(regs, TRUE);
switch (ec) {
case EC_UNKNOWN:
/* We dont expect to get this trap so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_WFI_WFE:
/* WFI emulation */
rc = cpu_vcpu_emulate_wfi_wfe(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP15_A32:
/* MCR/MRC CP15 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCRR_MRRC_CP15_A32:
/* MCRR/MRRC CP15 emulation */
rc = cpu_vcpu_emulate_mcrr_mrrc_cp15(vcpu, regs, il, iss);
break;
case EC_TRAP_MCR_MRC_CP14_A32:
/* MCR/MRC CP14 emulation */
rc = cpu_vcpu_emulate_mcr_mrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_LDC_STC_CP14_A32:
/* LDC/STC CP14 emulation */
rc = cpu_vcpu_emulate_ldc_stc_cp14(vcpu, regs, il, iss);
break;
case EC_SIMD_FPU:
/* Advanced SIMD and FPU emulation */
rc = cpu_vcpu_emulate_simd_fp_regs(vcpu, regs, il, iss);
break;
case EC_FPEXC_A32:
case EC_FPEXC_A64:
/* We dont expect any FP execution faults */
rc = VMM_EFAIL;
break;
case EC_TRAP_MRC_VMRS_CP10_A32:
/* MRC (or VMRS) to CP10 for MVFR0, MVFR1 or FPSID */
rc = cpu_vcpu_emulate_vmrs(vcpu, regs, il, iss);
break;
case EC_TRAP_MCRR_MRRC_CP14_A32:
/* MRRC to CP14 emulation */
rc = cpu_vcpu_emulate_mcrr_mrrc_cp14(vcpu, regs, il, iss);
break;
case EC_TRAP_SVC_A32:
case EC_TRAP_SVC_A64:
/* We dont expect to get these traps so error */
rc = VMM_EFAIL;
break;
case EC_TRAP_SMC_A32:
/* SMC emulation for A32 guest */
rc = cpu_vcpu_emulate_smc32(vcpu, regs, il, iss);
break;
case EC_TRAP_SMC_A64:
/* SMC emulation for A64 guest */
rc = cpu_vcpu_emulate_smc64(vcpu, regs, il, iss);
break;
case EC_TRAP_HVC_A32:
/* HVC emulation for A32 guest */
rc = cpu_vcpu_emulate_hvc32(vcpu, regs, il, iss);
break;
case EC_TRAP_HVC_A64:
/* HVC emulation for A64 guest */
rc = cpu_vcpu_emulate_hvc64(vcpu, regs, il, iss);
break;
case EC_TRAP_MSR_MRS_SYSTEM:
/* MSR/MRS/SystemRegs emulation */
rc = cpu_vcpu_emulate_msr_mrs_system(vcpu, regs, il, iss);
break;
case EC_TRAP_LWREL_INST_ABORT:
/* Stage2 instruction abort */
fipa = (mrs(hpfar_el2) & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (mrs(far_el2) & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_inst_abort(vcpu, regs, il, iss, fipa);
break;
case EC_TRAP_LWREL_DATA_ABORT:
/* Stage2 data abort */
fipa = (mrs(hpfar_el2) & HPFAR_FIPA_MASK) >> HPFAR_FIPA_SHIFT;
fipa = fipa << HPFAR_FIPA_PAGE_SHIFT;
fipa = fipa | (mrs(far_el2) & HPFAR_FIPA_PAGE_MASK);
rc = cpu_vcpu_data_abort(vcpu, regs, il, iss, fipa);
break;
case EC_CUREL_INST_ABORT:
case EC_CUREL_DATA_ABORT:
case EC_SERROR:
/* We dont expect to get aborts from EL2 so error */
rc = VMM_EFAIL;
break;
case EC_PC_UNALIGNED:
case EC_SP_UNALIGNED:
/* We dont expect to get alignment faults from EL2 */
rc = VMM_EFAIL;
break;
default:
/* Unhandled or unknown EC value so error */
rc = VMM_EFAIL;
break;
};
if (rc) {
vmm_printf("%s: CPU%d VCPU=%s sync failed (error %d)\n",
__func__, vmm_smp_processor_id(),
(vcpu) ? vcpu->name : "(NULL)", rc);
vmm_printf("%s: ESR=0x%016lx EC=0x%x IL=0x%x ISS=0x%x\n",
__func__, esr, ec, il, iss);
vmm_printf("%s: ELR=0x%016lx FAR=0x%016lx HPFAR=0x%016lx\n",
__func__, elr, far, mrs(hpfar_el2));
if (vmm_manager_vcpu_get_state(vcpu) != VMM_VCPU_STATE_HALTED) {
cpu_vcpu_halt(vcpu, regs);
}
}
vmm_scheduler_irq_exit(regs);
}
int arch_vcpu_init(struct vmm_vcpu *vcpu)
{
int rc = VMM_OK;
u32 cpuid = 0;
const char *attr;
irq_flags_t flags;
u32 saved_cptr_el2, saved_hstr_el2;
u32 phys_timer_irq, virt_timer_irq;
/* For both Orphan & Normal VCPUs */
memset(arm_regs(vcpu), 0, sizeof(arch_regs_t));
arm_regs(vcpu)->pc = vcpu->start_pc;
arm_regs(vcpu)->sp = vcpu->stack_va + vcpu->stack_sz - 8;
arm_regs(vcpu)->sp = arm_regs(vcpu)->sp & ~0x7;
if (!vcpu->is_normal) {
arm_regs(vcpu)->pstate = PSR_MODE64_EL2h;
arm_regs(vcpu)->pstate |= PSR_ASYNC_ABORT_DISABLED;
return VMM_OK;
}
/* Save CPTR_EL2 and HSTR_EL2 */
saved_cptr_el2 = mrs(cptr_el2);
saved_hstr_el2 = mrs(hstr_el2);
/* A VCPU running on different host CPU can be resetted
* using sync IPI. This means we can reach here while VCPU
* is running and coprocessor/system traps are enabled.
*
* We force disable coprocessor and system traps to ensure
* that we don't touch coprocessor and system registers
* while traps are enabled.
*/
msr(cptr_el2, 0x0);
msr(hstr_el2, 0x0);
/* Following initialization for normal VCPUs only */
rc = vmm_devtree_read_string(vcpu->node,
VMM_DEVTREE_COMPATIBLE_ATTR_NAME, &attr);
if (rc) {
goto done;
}
if (strcmp(attr, "armv7a,cortex-a8") == 0) {
cpuid = ARM_CPUID_CORTEXA8;
arm_regs(vcpu)->pstate = PSR_MODE32;
} else if (strcmp(attr, "armv7a,cortex-a9") == 0) {
cpuid = ARM_CPUID_CORTEXA9;
arm_regs(vcpu)->pstate = PSR_MODE32;
} else if (strcmp(attr, "armv7a,cortex-a15") == 0) {
cpuid = ARM_CPUID_CORTEXA15;
arm_regs(vcpu)->pstate = PSR_MODE32;
} else if (strcmp(attr, "armv7a,cortex-a7") == 0) {
cpuid = ARM_CPUID_CORTEXA7;
arm_regs(vcpu)->pstate = PSR_MODE32;
} else if (strcmp(attr, "armv8,generic") == 0) {
cpuid = ARM_CPUID_ARMV8;
} else {
rc = VMM_EINVALID;
goto done;
}
if (arm_regs(vcpu)->pstate == PSR_MODE32) {
/* Check if the host supports A32 mode @ EL1 */
if (!cpu_supports_el1_a32()) {
vmm_printf("Host does not support AArch32 mode\n");
rc = VMM_ENOTAVAIL;
goto done;
}
arm_regs(vcpu)->pstate |= PSR_ZERO_MASK;
arm_regs(vcpu)->pstate |= PSR_MODE32_SUPERVISOR;
} else {
arm_regs(vcpu)->pstate |= PSR_MODE64_DEBUG_DISABLED;
arm_regs(vcpu)->pstate |= PSR_MODE64_EL1h;
}
arm_regs(vcpu)->pstate |= PSR_ASYNC_ABORT_DISABLED;
arm_regs(vcpu)->pstate |= PSR_IRQ_DISABLED;
arm_regs(vcpu)->pstate |= PSR_FIQ_DISABLED;
/* First time initialization of private context */
if (!vcpu->reset_count) {
/* Alloc private context */
vcpu->arch_priv = vmm_zalloc(sizeof(struct arm_priv));
if (!vcpu->arch_priv) {
rc = VMM_ENOMEM;
goto done;
}
/* Setup CPUID value expected by VCPU in MIDR register
* as-per HW specifications.
*/
arm_priv(vcpu)->cpuid = cpuid;
/* Initialize VCPU features */
arm_priv(vcpu)->features = 0;
switch (cpuid) {
case ARM_CPUID_CORTEXA8:
arm_set_feature(vcpu, ARM_FEATURE_V7);
arm_set_feature(vcpu, ARM_FEATURE_VFP3);
arm_set_feature(vcpu, ARM_FEATURE_NEON);
arm_set_feature(vcpu, ARM_FEATURE_THUMB2EE);
arm_set_feature(vcpu, ARM_FEATURE_DUMMY_C15_REGS);
arm_set_feature(vcpu, ARM_FEATURE_TRUSTZONE);
break;
case ARM_CPUID_CORTEXA9:
arm_set_feature(vcpu, ARM_FEATURE_V7);
arm_set_feature(vcpu, ARM_FEATURE_VFP3);
arm_set_feature(vcpu, ARM_FEATURE_VFP_FP16);
arm_set_feature(vcpu, ARM_FEATURE_NEON);
arm_set_feature(vcpu, ARM_FEATURE_THUMB2EE);
arm_set_feature(vcpu, ARM_FEATURE_V7MP);
arm_set_feature(vcpu, ARM_FEATURE_TRUSTZONE);
break;
case ARM_CPUID_CORTEXA7:
arm_set_feature(vcpu, ARM_FEATURE_V7);
arm_set_feature(vcpu, ARM_FEATURE_VFP4);
arm_set_feature(vcpu, ARM_FEATURE_VFP_FP16);
arm_set_feature(vcpu, ARM_FEATURE_NEON);
arm_set_feature(vcpu, ARM_FEATURE_THUMB2EE);
arm_set_feature(vcpu, ARM_FEATURE_ARM_DIV);
arm_set_feature(vcpu, ARM_FEATURE_V7MP);
arm_set_feature(vcpu, ARM_FEATURE_GENERIC_TIMER);
arm_set_feature(vcpu, ARM_FEATURE_DUMMY_C15_REGS);
arm_set_feature(vcpu, ARM_FEATURE_LPAE);
arm_set_feature(vcpu, ARM_FEATURE_TRUSTZONE);
break;
case ARM_CPUID_CORTEXA15:
arm_set_feature(vcpu, ARM_FEATURE_V7);
arm_set_feature(vcpu, ARM_FEATURE_VFP4);
arm_set_feature(vcpu, ARM_FEATURE_VFP_FP16);
arm_set_feature(vcpu, ARM_FEATURE_NEON);
arm_set_feature(vcpu, ARM_FEATURE_THUMB2EE);
arm_set_feature(vcpu, ARM_FEATURE_ARM_DIV);
arm_set_feature(vcpu, ARM_FEATURE_V7MP);
arm_set_feature(vcpu, ARM_FEATURE_GENERIC_TIMER);
arm_set_feature(vcpu, ARM_FEATURE_DUMMY_C15_REGS);
arm_set_feature(vcpu, ARM_FEATURE_LPAE);
arm_set_feature(vcpu, ARM_FEATURE_TRUSTZONE);
break;
case ARM_CPUID_ARMV8:
arm_set_feature(vcpu, ARM_FEATURE_V8);
arm_set_feature(vcpu, ARM_FEATURE_VFP4);
arm_set_feature(vcpu, ARM_FEATURE_ARM_DIV);
arm_set_feature(vcpu, ARM_FEATURE_LPAE);
arm_set_feature(vcpu, ARM_FEATURE_GENERIC_TIMER);
break;
default:
break;
};
/* Some features automatically imply others: */
if (arm_feature(vcpu, ARM_FEATURE_V7)) {
arm_set_feature(vcpu, ARM_FEATURE_VAPA);
arm_set_feature(vcpu, ARM_FEATURE_THUMB2);
arm_set_feature(vcpu, ARM_FEATURE_MPIDR);
if (!arm_feature(vcpu, ARM_FEATURE_M)) {
arm_set_feature(vcpu, ARM_FEATURE_V6K);
} else {
arm_set_feature(vcpu, ARM_FEATURE_V6);
}
}
if (arm_feature(vcpu, ARM_FEATURE_V6K)) {
arm_set_feature(vcpu, ARM_FEATURE_V6);
arm_set_feature(vcpu, ARM_FEATURE_MVFR);
}
if (arm_feature(vcpu, ARM_FEATURE_V6)) {
arm_set_feature(vcpu, ARM_FEATURE_V5);
if (!arm_feature(vcpu, ARM_FEATURE_M)) {
arm_set_feature(vcpu, ARM_FEATURE_AUXCR);
}
}
if (arm_feature(vcpu, ARM_FEATURE_V5)) {
arm_set_feature(vcpu, ARM_FEATURE_V4T);
}
if (arm_feature(vcpu, ARM_FEATURE_M)) {
arm_set_feature(vcpu, ARM_FEATURE_THUMB_DIV);
}
if (arm_feature(vcpu, ARM_FEATURE_ARM_DIV)) {
arm_set_feature(vcpu, ARM_FEATURE_THUMB_DIV);
}
if (arm_feature(vcpu, ARM_FEATURE_VFP4)) {
arm_set_feature(vcpu, ARM_FEATURE_VFP3);
}
if (arm_feature(vcpu, ARM_FEATURE_VFP3)) {
arm_set_feature(vcpu, ARM_FEATURE_VFP);
}
if (arm_feature(vcpu, ARM_FEATURE_LPAE)) {
arm_set_feature(vcpu, ARM_FEATURE_PXN);
}
/* Initialize Hypervisor Configuration */
INIT_SPIN_LOCK(&arm_priv(vcpu)->hcr_lock);
arm_priv(vcpu)->hcr = (HCR_TSW_MASK |
HCR_TACR_MASK |
HCR_TIDCP_MASK |
HCR_TSC_MASK |
HCR_TWE_MASK |
HCR_TWI_MASK |
HCR_AMO_MASK |
HCR_IMO_MASK |
HCR_FMO_MASK |
HCR_SWIO_MASK |
HCR_VM_MASK);
if (!(arm_regs(vcpu)->pstate & PSR_MODE32)) {
arm_priv(vcpu)->hcr |= HCR_RW_MASK;
}
/* Initialize Coprocessor Trap Register */
arm_priv(vcpu)->cptr = CPTR_TTA_MASK;
arm_priv(vcpu)->cptr |= CPTR_TFP_MASK;
/* Initialize Hypervisor System Trap Register */
arm_priv(vcpu)->hstr = 0;
/* Cleanup VGIC context first time */
arm_vgic_cleanup(vcpu);
}
/* Clear virtual exception bits in HCR */
vmm_spin_lock_irqsave(&arm_priv(vcpu)->hcr_lock, flags);
arm_priv(vcpu)->hcr &= ~(HCR_VSE_MASK |
HCR_VI_MASK |
HCR_VF_MASK);
vmm_spin_unlock_irqrestore(&arm_priv(vcpu)->hcr_lock, flags);
/* Set last host CPU to invalid value */
arm_priv(vcpu)->last_hcpu = 0xFFFFFFFF;
/* Initialize sysregs context */
rc = cpu_vcpu_sysregs_init(vcpu, cpuid);
if (rc) {
goto fail_sysregs_init;
}
/* Initialize VFP context */
rc = cpu_vcpu_vfp_init(vcpu);
if (rc) {
goto fail_vfp_init;
}
/* Initialize generic timer context */
if (arm_feature(vcpu, ARM_FEATURE_GENERIC_TIMER)) {
if (vmm_devtree_read_u32(vcpu->node,
"gentimer_phys_irq",
&phys_timer_irq)) {
phys_timer_irq = 0;
}
if (vmm_devtree_read_u32(vcpu->node,
"gentimer_virt_irq",
&virt_timer_irq)) {
virt_timer_irq = 0;
}
rc = generic_timer_vcpu_context_init(vcpu,
&arm_gentimer_context(vcpu),
phys_timer_irq,
virt_timer_irq);
if (rc) {
goto fail_gentimer_init;
}
}
rc = VMM_OK;
goto done;
fail_gentimer_init:
if (!vcpu->reset_count) {
cpu_vcpu_vfp_deinit(vcpu);
}
fail_vfp_init:
if (!vcpu->reset_count) {
cpu_vcpu_sysregs_deinit(vcpu);
}
fail_sysregs_init:
if (!vcpu->reset_count) {
vmm_free(vcpu->arch_priv);
vcpu->arch_priv = NULL;
}
done:
msr(cptr_el2, saved_cptr_el2);
msr(hstr_el2, saved_hstr_el2);
return rc;
}
