/* Set the timer to wake us up at a particular time.
* Timeout is a Xen system time (nanoseconds since boot); 0 disables the timer.
* Returns 1 on success; 0 if the timeout is too soon or is in the past. */
int reprogram_timer(s_time_t timeout)
{
uint64_t deadline;
if ( timeout == 0 )
{
#if USE_HYP_TIMER
WRITE_SYSREG32(0, CNTHP_CTL_EL2);
#else
WRITE_SYSREG32(0, CNTP_CTL_EL0);
#endif
return 1;
}
deadline = ns_to_ticks(timeout) + boot_count;
#if USE_HYP_TIMER
WRITE_SYSREG64(deadline, CNTHP_CVAL_EL2);
WRITE_SYSREG32(CNTx_CTL_ENABLE, CNTHP_CTL_EL2);
#else
WRITE_SYSREG64(deadline, CNTP_CVAL_EL0);
WRITE_SYSREG32(CNTx_CTL_ENABLE, CNTP_CTL_EL0);
#endif
isb();
/* No need to check for timers in the past; the Generic Timer fires
* on a signed 63-bit comparison. */
return 1;
}
/* Set up the timer interrupt on this CPU */
void __cpuinit init_timer_interrupt(void)
{
/* Sensible defaults */
WRITE_SYSREG64(0, CNTVOFF_EL2); /* No VM-specific offset */
/* Do not let the VMs program the physical timer, only read the physical counter */
WRITE_SYSREG32(CNTHCTL_PA, CNTHCTL_EL2);
WRITE_SYSREG32(0, CNTP_CTL_EL0); /* Physical timer disabled */
WRITE_SYSREG32(0, CNTHP_CTL_EL2); /* Hypervisor's timer disabled */
isb();
request_irq(timer_irq[TIMER_HYP_PPI], 0, timer_interrupt,
"hyptimer", NULL);
request_irq(timer_irq[TIMER_VIRT_PPI], 0, vtimer_interrupt,
"virtimer", NULL);
request_irq(timer_irq[TIMER_PHYS_NONSECURE_PPI], 0, timer_interrupt,
"phytimer", NULL);
}
/* Handle the firing timer */
static void timer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
if ( irq == (timer_irq[TIMER_HYP_PPI].irq) &&
READ_SYSREG32(CNTHP_CTL_EL2) & CNTx_CTL_PENDING )
{
/* Signal the generic timer code to do its work */
raise_softirq(TIMER_SOFTIRQ);
/* Disable the timer to avoid more interrupts */
WRITE_SYSREG32(0, CNTHP_CTL_EL2);
}
if ( irq == (timer_irq[TIMER_PHYS_NONSECURE_PPI].irq) &&
READ_SYSREG32(CNTP_CTL_EL0) & CNTx_CTL_PENDING )
{
/* Signal the generic timer code to do its work */
raise_softirq(TIMER_SOFTIRQ);
/* Disable the timer to avoid more interrupts */
WRITE_SYSREG32(0, CNTP_CTL_EL0);
}
}
/* Very early check of the CPU cache properties */
void __init setup_cache(void)
{
uint32_t ccsid;
/* Read the cache size ID register for the level-0 data cache */
WRITE_SYSREG32(0, CSSELR_EL1);
ccsid = READ_SYSREG32(CCSIDR_EL1);
/* Low 3 bits are log2(cacheline size in words) - 2. */
cacheline_bytes = 1U << (4 + (ccsid & 0x7));
}
int virt_timer_restore(struct vcpu *v)
{
if ( is_idle_domain(v->domain) )
return 0;
stop_timer(&v->arch.virt_timer.timer);
WRITE_SYSREG64(v->arch.virt_timer.offset, CNTVOFF_EL2);
WRITE_SYSREG64(v->arch.virt_timer.cval, CNTV_CVAL_EL0);
WRITE_SYSREG32(v->arch.virt_timer.ctl, CNTV_CTL_EL0);
return 0;
}
int virt_timer_restore(struct vcpu *v)
{
ASSERT(!is_idle_vcpu(v));
stop_timer(&v->arch.virt_timer.timer);
migrate_timer(&v->arch.virt_timer.timer, v->processor);
migrate_timer(&v->arch.phys_timer.timer, v->processor);
WRITE_SYSREG64(v->domain->arch.virt_timer_base.offset, CNTVOFF_EL2);
WRITE_SYSREG64(v->arch.virt_timer.cval, CNTV_CVAL_EL0);
WRITE_SYSREG32(v->arch.virt_timer.ctl, CNTV_CTL_EL0);
return 0;
}
/* Set up the timer interrupt on this CPU */
void __cpuinit init_timer_interrupt(void)
{
/* Sensible defaults */
WRITE_SYSREG64(0, CNTVOFF_EL2); /* No VM-specific offset */
WRITE_SYSREG32(0, CNTKCTL_EL1); /* No user-mode access */
#if USE_HYP_TIMER
/* Do not let the VMs program the physical timer, only read the physical counter */
WRITE_SYSREG32(CNTHCTL_PA, CNTHCTL_EL2);
#else
/* Cannot let VMs access physical counter if we are using it */
WRITE_SYSREG32(0, CNTHCTL_EL2);
#endif
WRITE_SYSREG32(0, CNTP_CTL_EL0); /* Physical timer disabled */
WRITE_SYSREG32(0, CNTHP_CTL_EL2); /* Hypervisor's timer disabled */
isb();
request_dt_irq(&timer_irq[TIMER_HYP_PPI], timer_interrupt,
"hyptimer", NULL);
request_dt_irq(&timer_irq[TIMER_VIRT_PPI], vtimer_interrupt,
"virtimer", NULL);
request_dt_irq(&timer_irq[TIMER_PHYS_NONSECURE_PPI], timer_interrupt,
"phytimer", NULL);
}
int virt_timer_save(struct vcpu *v)
{
ASSERT(!is_idle_vcpu(v));
v->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(v->arch.virt_timer.ctl & ~CNTx_CTL_ENABLE, CNTV_CTL_EL0);
v->arch.virt_timer.cval = READ_SYSREG64(CNTV_CVAL_EL0);
if ( (v->arch.virt_timer.ctl & CNTx_CTL_ENABLE) &&
!(v->arch.virt_timer.ctl & CNTx_CTL_MASK))
{
set_timer(&v->arch.virt_timer.timer, ticks_to_ns(v->arch.virt_timer.cval +
v->domain->arch.virt_timer_base.offset - boot_count));
}
return 0;
}
int virt_timer_save(struct vcpu *v)
{
if ( is_idle_domain(v->domain) )
return 0;
v->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(v->arch.virt_timer.ctl & ~CNTx_CTL_ENABLE, CNTV_CTL_EL0);
v->arch.virt_timer.cval = READ_SYSREG64(CNTV_CVAL_EL0);
if ( v->arch.virt_timer.ctl & CNTx_CTL_ENABLE )
{
set_timer(&v->arch.virt_timer.timer, ticks_to_ns(v->arch.virt_timer.cval +
v->arch.virt_timer.offset - boot_count));
}
return 0;
}
static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
/*
* Edge-triggered interrupts can be used for the virtual timer. Even
* if the timer output signal is masked in the context switch, the
* GIC will keep track that of any interrupts raised while IRQS are
* disabled. As soon as IRQs are re-enabled, the virtual interrupt
* will be injected to Xen.
*
* If an IDLE vCPU was scheduled next then we should ignore the
* interrupt.
*/
if ( unlikely(is_idle_vcpu(current)) )
return;
current->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(current->arch.virt_timer.ctl | CNTx_CTL_MASK, CNTV_CTL_EL0);
vgic_vcpu_inject_irq(current, current->arch.virt_timer.irq);
}
static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
current->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(current->arch.virt_timer.ctl | CNTx_CTL_MASK, CNTV_CTL_EL0);
vgic_vcpu_inject_irq(current, current->arch.virt_timer.irq, 1);
}
static void ctxt_switch_to(struct vcpu *n)
{
p2m_restore_state(n);
WRITE_SYSREG32(n->domain->arch.vpidr, VPIDR_EL2);
WRITE_SYSREG(n->arch.vmpidr, VMPIDR_EL2);
/* VGIC */
gic_restore_state(n);
/* VFP */
vfp_restore_state(n);
/* XXX MPU */
/* Fault Status */
#if defined(CONFIG_ARM_32)
WRITE_CP32(n->arch.dfar, DFAR);
WRITE_CP32(n->arch.ifar, IFAR);
WRITE_CP32(n->arch.dfsr, DFSR);
#elif defined(CONFIG_ARM_64)
WRITE_SYSREG64(n->arch.far, FAR_EL1);
WRITE_SYSREG64(n->arch.esr, ESR_EL1);
#endif
if ( is_32bit_domain(n->domain) )
WRITE_SYSREG(n->arch.ifsr, IFSR32_EL2);
WRITE_SYSREG(n->arch.afsr0, AFSR0_EL1);
WRITE_SYSREG(n->arch.afsr1, AFSR1_EL1);
/* MMU */
WRITE_SYSREG(n->arch.vbar, VBAR_EL1);
WRITE_SYSREG(n->arch.ttbcr, TCR_EL1);
WRITE_SYSREG64(n->arch.ttbr0, TTBR0_EL1);
WRITE_SYSREG64(n->arch.ttbr1, TTBR1_EL1);
if ( is_32bit_domain(n->domain) )
WRITE_SYSREG(n->arch.dacr, DACR32_EL2);
WRITE_SYSREG64(n->arch.par, PAR_EL1);
#if defined(CONFIG_ARM_32)
WRITE_CP32(n->arch.mair0, MAIR0);
WRITE_CP32(n->arch.mair1, MAIR1);
WRITE_CP32(n->arch.amair0, AMAIR0);
WRITE_CP32(n->arch.amair1, AMAIR1);
#elif defined(CONFIG_ARM_64)
WRITE_SYSREG64(n->arch.mair, MAIR_EL1);
WRITE_SYSREG64(n->arch.amair, AMAIR_EL1);
#endif
isb();
/* Control Registers */
WRITE_SYSREG(n->arch.cpacr, CPACR_EL1);
WRITE_SYSREG(n->arch.contextidr, CONTEXTIDR_EL1);
WRITE_SYSREG(n->arch.tpidr_el0, TPIDR_EL0);
WRITE_SYSREG(n->arch.tpidrro_el0, TPIDRRO_EL0);
WRITE_SYSREG(n->arch.tpidr_el1, TPIDR_EL1);
if ( is_32bit_domain(n->domain) && cpu_has_thumbee )
{
WRITE_SYSREG32(n->arch.teecr, TEECR32_EL1);
WRITE_SYSREG32(n->arch.teehbr, TEEHBR32_EL1);
}
#ifdef CONFIG_ARM_32
WRITE_CP32(n->arch.joscr, JOSCR);
WRITE_CP32(n->arch.jmcr, JMCR);
#endif
isb();
/* CP 15 */
WRITE_SYSREG(n->arch.csselr, CSSELR_EL1);
isb();
/* This is could trigger an hardware interrupt from the virtual
* timer. The interrupt needs to be injected into the guest. */
WRITE_SYSREG32(n->arch.cntkctl, CNTKCTL_EL1);
virt_timer_restore(n);
}
/* Inject an undefined exception into a 64 bit guest */
static void inject_undef64_exception(struct cpu_user_regs *regs, int instr_len)
{
union hsr esr = {
.iss = 0,
.len = instr_len,
.ec = HSR_EC_UNKNOWN,
};
BUG_ON( is_pv32_domain(current->domain) );
regs->spsr_el1 = regs->cpsr;
regs->elr_el1 = regs->pc;
regs->cpsr = PSR_MODE_EL1h | PSR_ABT_MASK | PSR_FIQ_MASK | \
PSR_IRQ_MASK | PSR_DBG_MASK;
regs->pc = READ_SYSREG(VBAR_EL1) + VECTOR64_CURRENT_SPx_SYNC;
WRITE_SYSREG32(esr.bits, ESR_EL1);
}
/* Inject an abort exception into a 64 bit guest */
static void inject_abt64_exception(struct cpu_user_regs *regs,
int prefetch,
register_t addr,
int instr_len)
{
union hsr esr = {
.iss = 0,
.len = instr_len,
};
/*
* Trap may have been taken from EL0, which might be in AArch32
* mode (PSR_MODE_BIT set), or in AArch64 mode (PSR_MODE_EL0t).
*
* Since we know the kernel must be 64-bit any trap from a 32-bit
* mode must have been from EL0.
*/
if ( psr_mode_is_32bit(regs->cpsr) || psr_mode(regs->cpsr,PSR_MODE_EL0t) )
esr.ec = prefetch
? HSR_EC_INSTR_ABORT_LOWER_EL : HSR_EC_DATA_ABORT_LOWER_EL;
else
esr.ec = prefetch
? HSR_EC_INSTR_ABORT_CURR_EL : HSR_EC_DATA_ABORT_CURR_EL;
BUG_ON( is_pv32_domain(current->domain) );
regs->spsr_el1 = regs->cpsr;
regs->elr_el1 = regs->pc;
regs->cpsr = PSR_MODE_EL1h | PSR_ABT_MASK | PSR_FIQ_MASK | \
PSR_IRQ_MASK | PSR_DBG_MASK;
regs->pc = READ_SYSREG(VBAR_EL1) + VECTOR64_CURRENT_SPx_SYNC;
WRITE_SYSREG(addr, FAR_EL1);
WRITE_SYSREG32(esr.bits, ESR_EL1);
}
static void inject_dabt64_exception(struct cpu_user_regs *regs,
register_t addr,
int instr_len)
{
inject_abt64_exception(regs, 0, addr, instr_len);
}
static void inject_iabt64_exception(struct cpu_user_regs *regs,
register_t addr,
int instr_len)
{
inject_abt64_exception(regs, 1, addr, instr_len);
}
static void ctxt_switch_to(struct vcpu *n)
{
/* When the idle VCPU is running, Xen will always stay in hypervisor
* mode. Therefore we don't need to restore the context of an idle VCPU.
*/
if ( is_idle_vcpu(n) )
return;
p2m_restore_state(n);
WRITE_SYSREG32(n->domain->arch.vpidr, VPIDR_EL2);
WRITE_SYSREG(n->arch.vmpidr, VMPIDR_EL2);
/* VGIC */
gic_restore_state(n);
/* VFP */
vfp_restore_state(n);
/* XXX MPU */
/* Fault Status */
#if defined(CONFIG_ARM_32)
WRITE_CP32(n->arch.dfar, DFAR);
WRITE_CP32(n->arch.ifar, IFAR);
WRITE_CP32(n->arch.dfsr, DFSR);
#elif defined(CONFIG_ARM_64)
WRITE_SYSREG64(n->arch.far, FAR_EL1);
WRITE_SYSREG64(n->arch.esr, ESR_EL1);
#endif
if ( is_32bit_domain(n->domain) )
WRITE_SYSREG(n->arch.ifsr, IFSR32_EL2);
WRITE_SYSREG(n->arch.afsr0, AFSR0_EL1);
WRITE_SYSREG(n->arch.afsr1, AFSR1_EL1);
/* MMU */
WRITE_SYSREG(n->arch.vbar, VBAR_EL1);
WRITE_SYSREG(n->arch.ttbcr, TCR_EL1);
WRITE_SYSREG64(n->arch.ttbr0, TTBR0_EL1);
WRITE_SYSREG64(n->arch.ttbr1, TTBR1_EL1);
/*
* Erratum #852523: DACR32_EL2 must be restored before one of the
* following sysregs: SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1 or
* CONTEXTIDR_EL1.
*/
if ( is_32bit_domain(n->domain) )
WRITE_SYSREG(n->arch.dacr, DACR32_EL2);
WRITE_SYSREG64(n->arch.par, PAR_EL1);
#if defined(CONFIG_ARM_32)
WRITE_CP32(n->arch.mair0, MAIR0);
WRITE_CP32(n->arch.mair1, MAIR1);
WRITE_CP32(n->arch.amair0, AMAIR0);
WRITE_CP32(n->arch.amair1, AMAIR1);
#elif defined(CONFIG_ARM_64)
WRITE_SYSREG64(n->arch.mair, MAIR_EL1);
WRITE_SYSREG64(n->arch.amair, AMAIR_EL1);
#endif
isb();
/* Control Registers */
WRITE_SYSREG(n->arch.cpacr, CPACR_EL1);
/*
* This write to sysreg CONTEXTIDR_EL1 ensures we don't hit erratum
* #852523. I.e DACR32_EL2 is not correctly synchronized.
*/
WRITE_SYSREG(n->arch.contextidr, CONTEXTIDR_EL1);
WRITE_SYSREG(n->arch.tpidr_el0, TPIDR_EL0);
WRITE_SYSREG(n->arch.tpidrro_el0, TPIDRRO_EL0);
WRITE_SYSREG(n->arch.tpidr_el1, TPIDR_EL1);
if ( is_32bit_domain(n->domain) && cpu_has_thumbee )
{
WRITE_SYSREG32(n->arch.teecr, TEECR32_EL1);
WRITE_SYSREG32(n->arch.teehbr, TEEHBR32_EL1);
}
#ifdef CONFIG_ARM_32
WRITE_CP32(n->arch.joscr, JOSCR);
WRITE_CP32(n->arch.jmcr, JMCR);
#endif
isb();
/* CP 15 */
WRITE_SYSREG(n->arch.csselr, CSSELR_EL1);
isb();
/* This is could trigger an hardware interrupt from the virtual
* timer. The interrupt needs to be injected into the guest. */
WRITE_SYSREG32(n->arch.cntkctl, CNTKCTL_EL1);
virt_timer_restore(n);
}
/* MMU setup for secondary CPUS (which already have paging enabled) */
void __cpuinit mmu_init_secondary_cpu(void)
{
/* From now on, no mapping may be both writable and executable. */
WRITE_SYSREG32(READ_SYSREG32(SCTLR_EL2) | SCTLR_WXN, SCTLR_EL2);
flush_xen_text_tlb();
}
/* Boot-time pagetable setup.
* Changes here may need matching changes in head.S */
void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
{
unsigned long dest_va;
lpae_t pte, *p;
int i;
/* Map the destination in the boot misc area. */
dest_va = BOOT_MISC_VIRT_START;
pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT);
write_pte(xen_second + second_table_offset(dest_va), pte);
flush_xen_data_tlb_range_va(dest_va, SECOND_SIZE);
/* Calculate virt-to-phys offset for the new location */
phys_offset = xen_paddr - (unsigned long) _start;
/* Copy */
memcpy((void *) dest_va, _start, _end - _start);
/* Beware! Any state we modify between now and the PT switch may be
* discarded when we switch over to the copy. */
/* Update the copy of xen_pgtable to use the new paddrs */
p = (void *) xen_pgtable + dest_va - (unsigned long) _start;
#ifdef CONFIG_ARM_64
p[0].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
p = (void *) xen_first + dest_va - (unsigned long) _start;
#endif
for ( i = 0; i < 4; i++)
p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
p = (void *) xen_second + dest_va - (unsigned long) _start;
if ( boot_phys_offset != 0 )
{
/* Remove the old identity mapping of the boot paddr */
vaddr_t va = (vaddr_t)_start + boot_phys_offset;
p[second_linear_offset(va)].bits = 0;
}
for ( i = 0; i < 4 * LPAE_ENTRIES; i++)
if ( p[i].pt.valid )
p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
/* Change pagetables to the copy in the relocated Xen */
boot_ttbr = (uintptr_t) xen_pgtable + phys_offset;
flush_xen_dcache(boot_ttbr);
flush_xen_dcache_va_range((void*)dest_va, _end - _start);
flush_xen_text_tlb();
WRITE_SYSREG64(boot_ttbr, TTBR0_EL2);
dsb(); /* Ensure visibility of HTTBR update */
flush_xen_text_tlb();
/* Undo the temporary map */
pte.bits = 0;
write_pte(xen_second + second_table_offset(dest_va), pte);
flush_xen_text_tlb();
/* Link in the fixmap pagetable */
pte = mfn_to_xen_entry((((unsigned long) xen_fixmap) + phys_offset)
>> PAGE_SHIFT);
pte.pt.table = 1;
write_pte(xen_second + second_table_offset(FIXMAP_ADDR(0)), pte);
/*
* No flush required here. Individual flushes are done in
* set_fixmap as entries are used.
*/
/* Break up the Xen mapping into 4k pages and protect them separately. */
for ( i = 0; i < LPAE_ENTRIES; i++ )
{
unsigned long mfn = paddr_to_pfn(xen_paddr) + i;
unsigned long va = XEN_VIRT_START + (i << PAGE_SHIFT);
if ( !is_kernel(va) )
break;
pte = mfn_to_xen_entry(mfn);
pte.pt.table = 1; /* 4k mappings always have this bit set */
if ( is_kernel_text(va) || is_kernel_inittext(va) )
{
pte.pt.xn = 0;
pte.pt.ro = 1;
}
if ( is_kernel_rodata(va) )
pte.pt.ro = 1;
write_pte(xen_xenmap + i, pte);
/* No flush required here as page table is not hooked in yet. */
}
pte = mfn_to_xen_entry((((unsigned long) xen_xenmap) + phys_offset)
>> PAGE_SHIFT);
pte.pt.table = 1;
write_pte(xen_second + second_linear_offset(XEN_VIRT_START), pte);
/* TLBFLUSH and ISB would be needed here, but wait until we set WXN */
/* From now on, no mapping may be both writable and executable. */
WRITE_SYSREG32(READ_SYSREG32(SCTLR_EL2) | SCTLR_WXN, SCTLR_EL2);
/* Flush everything after setting WXN bit. */
flush_xen_text_tlb();
}