/* Set up the timer on the boot CPU (early init function) */
void __init preinit_xen_time(void)
{
static const struct dt_device_match timer_ids[] __initconst =
{
DT_MATCH_TIMER,
{ /* sentinel */ },
};
int res;
u32 rate;
timer = dt_find_matching_node(NULL, timer_ids);
if ( !timer )
panic("Unable to find a compatible timer in the device tree");
dt_device_set_used_by(timer, DOMID_XEN);
res = platform_init_time();
if ( res )
panic("Timer: Cannot initialize platform timer");
res = dt_property_read_u32(timer, "clock-frequency", &rate);
if ( res )
cpu_khz = rate / 1000;
else
cpu_khz = READ_SYSREG32(CNTFRQ_EL0) / 1000;
boot_count = READ_SYSREG64(CNTPCT_EL0);
}
static vaddr_t exception_handler(vaddr_t offset)
{
uint32_t sctlr = READ_SYSREG32(SCTLR_EL1);
if (sctlr & SCTLR_V)
return 0xffff0000 + offset;
else /* always have security exceptions */
return READ_SYSREG(VBAR_EL1) + offset;
}
/* Handle the firing timer */
static void timer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
if ( irq == (timer_irq[TIMER_HYP_PPI].irq) &&
READ_SYSREG32(CNTHP_CTL_EL2) & CNTx_CTL_PENDING )
{
/* Signal the generic timer code to do its work */
raise_softirq(TIMER_SOFTIRQ);
/* Disable the timer to avoid more interrupts */
WRITE_SYSREG32(0, CNTHP_CTL_EL2);
}
if ( irq == (timer_irq[TIMER_PHYS_NONSECURE_PPI].irq) &&
READ_SYSREG32(CNTP_CTL_EL0) & CNTx_CTL_PENDING )
{
/* Signal the generic timer code to do its work */
raise_softirq(TIMER_SOFTIRQ);
/* Disable the timer to avoid more interrupts */
WRITE_SYSREG32(0, CNTP_CTL_EL0);
}
}
/* Very early check of the CPU cache properties */
void __init setup_cache(void)
{
uint32_t ccsid;
/* Read the cache size ID register for the level-0 data cache */
WRITE_SYSREG32(0, CSSELR_EL1);
ccsid = READ_SYSREG32(CCSIDR_EL1);
/* Low 3 bits are log2(cacheline size in words) - 2. */
cacheline_bytes = 1U << (4 + (ccsid & 0x7));
}
/* Set up the timer on the boot CPU */
int __init init_xen_time(void)
{
static const struct dt_device_match timer_ids[] __initconst =
{
DT_MATCH_TIMER,
{ /* sentinel */ },
};
struct dt_device_node *dev;
int res;
unsigned int i;
u32 rate;
dev = dt_find_matching_node(NULL, timer_ids);
if ( !dev )
panic("Unable to find a compatible timer in the device tree");
dt_device_set_used_by(dev, DOMID_XEN);
/* Retrieve all IRQs for the timer */
for ( i = TIMER_PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++ )
{
res = platform_get_irq(dev, i);
if ( res < 0 )
panic("Timer: Unable to retrieve IRQ %u from the device tree", i);
timer_irq[i] = res;
}
printk("Generic Timer IRQ: phys=%u hyp=%u virt=%u\n",
timer_irq[TIMER_PHYS_NONSECURE_PPI],
timer_irq[TIMER_HYP_PPI],
timer_irq[TIMER_VIRT_PPI]);
res = platform_init_time();
if ( res )
panic("Timer: Cannot initialize platform timer");
/* Check that this CPU supports the Generic Timer interface */
if ( !cpu_has_gentimer )
panic("CPU does not support the Generic Timer v1 interface");
res = dt_property_read_u32(dev, "clock-frequency", &rate);
if ( res )
cpu_khz = rate / 1000;
else
cpu_khz = READ_SYSREG32(CNTFRQ_EL0) / 1000;
boot_count = READ_SYSREG64(CNTPCT_EL0);
printk("Using generic timer at %lu KHz\n", cpu_khz);
return 0;
}
static void cpsr_switch_mode(struct cpu_user_regs *regs, int mode)
{
uint32_t sctlr = READ_SYSREG32(SCTLR_EL1);
regs->cpsr &= ~(PSR_MODE_MASK|PSR_IT_MASK|PSR_JAZELLE|PSR_BIG_ENDIAN|PSR_THUMB);
regs->cpsr |= mode;
regs->cpsr |= PSR_IRQ_MASK;
if ( mode == PSR_MODE_ABT )
regs->cpsr |= PSR_ABT_MASK;
if ( sctlr & SCTLR_TE )
regs->cpsr |= PSR_THUMB;
if ( sctlr & SCTLR_EE )
regs->cpsr |= PSR_BIG_ENDIAN;
}
int virt_timer_save(struct vcpu *v)
{
ASSERT(!is_idle_vcpu(v));
v->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(v->arch.virt_timer.ctl & ~CNTx_CTL_ENABLE, CNTV_CTL_EL0);
v->arch.virt_timer.cval = READ_SYSREG64(CNTV_CVAL_EL0);
if ( (v->arch.virt_timer.ctl & CNTx_CTL_ENABLE) &&
!(v->arch.virt_timer.ctl & CNTx_CTL_MASK))
{
set_timer(&v->arch.virt_timer.timer, ticks_to_ns(v->arch.virt_timer.cval +
v->domain->arch.virt_timer_base.offset - boot_count));
}
return 0;
}
int virt_timer_save(struct vcpu *v)
{
if ( is_idle_domain(v->domain) )
return 0;
v->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(v->arch.virt_timer.ctl & ~CNTx_CTL_ENABLE, CNTV_CTL_EL0);
v->arch.virt_timer.cval = READ_SYSREG64(CNTV_CVAL_EL0);
if ( v->arch.virt_timer.ctl & CNTx_CTL_ENABLE )
{
set_timer(&v->arch.virt_timer.timer, ticks_to_ns(v->arch.virt_timer.cval +
v->arch.virt_timer.offset - boot_count));
}
return 0;
}
int vcpu_initialise(struct vcpu *v)
{
int rc = 0;
BUILD_BUG_ON( sizeof(struct cpu_info) > STACK_SIZE );
v->arch.stack = alloc_xenheap_pages(STACK_ORDER, MEMF_node(vcpu_to_node(v)));
if ( v->arch.stack == NULL )
return -ENOMEM;
v->arch.cpu_info = (struct cpu_info *)(v->arch.stack
+ STACK_SIZE
- sizeof(struct cpu_info));
memset(&v->arch.saved_context, 0, sizeof(v->arch.saved_context));
v->arch.saved_context.sp = (register_t)v->arch.cpu_info;
v->arch.saved_context.pc = (register_t)continue_new_vcpu;
/* Idle VCPUs don't need the rest of this setup */
if ( is_idle_vcpu(v) )
return rc;
v->arch.sctlr = SCTLR_GUEST_INIT;
/*
* By default exposes an SMP system with AFF0 set to the VCPU ID
* TODO: Handle multi-threading processor and cluster
*/
v->arch.vmpidr = MPIDR_SMP | (v->vcpu_id << MPIDR_AFF0_SHIFT);
v->arch.actlr = READ_SYSREG32(ACTLR_EL1);
processor_vcpu_initialise(v);
if ( (rc = vcpu_vgic_init(v)) != 0 )
goto fail;
if ( (rc = vcpu_vtimer_init(v)) != 0 )
goto fail;
return rc;
fail:
vcpu_destroy(v);
return rc;
}
void __init preinit_xen_time(void)
{
int res;
/* Initialize all the generic timers presented in GTDT */
if ( acpi_disabled )
preinit_dt_xen_time();
else
preinit_acpi_xen_time();
if ( !cpu_khz )
cpu_khz = READ_SYSREG32(CNTFRQ_EL0) / 1000;
res = platform_init_time();
if ( res )
panic("Timer: Cannot initialize platform timer");
boot_count = READ_SYSREG64(CNTPCT_EL0);
}
static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
/*
* Edge-triggered interrupts can be used for the virtual timer. Even
* if the timer output signal is masked in the context switch, the
* GIC will keep track that of any interrupts raised while IRQS are
* disabled. As soon as IRQs are re-enabled, the virtual interrupt
* will be injected to Xen.
*
* If an IDLE vCPU was scheduled next then we should ignore the
* interrupt.
*/
if ( unlikely(is_idle_vcpu(current)) )
return;
current->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(current->arch.virt_timer.ctl | CNTx_CTL_MASK, CNTV_CTL_EL0);
vgic_vcpu_inject_irq(current, current->arch.virt_timer.irq);
}
int vcpu_initialise(struct vcpu *v)
{
int rc = 0;
BUILD_BUG_ON( sizeof(struct cpu_info) > STACK_SIZE );
v->arch.stack = alloc_xenheap_pages(STACK_ORDER, MEMF_node(vcpu_to_node(v)));
if ( v->arch.stack == NULL )
return -ENOMEM;
v->arch.cpu_info = (struct cpu_info *)(v->arch.stack
+ STACK_SIZE
- sizeof(struct cpu_info));
memset(&v->arch.saved_context, 0, sizeof(v->arch.saved_context));
v->arch.saved_context.sp = (register_t)v->arch.cpu_info;
v->arch.saved_context.pc = (register_t)continue_new_vcpu;
/* Idle VCPUs don't need the rest of this setup */
if ( is_idle_vcpu(v) )
return rc;
v->arch.sctlr = SCTLR_GUEST_INIT;
v->arch.vmpidr = MPIDR_SMP | vcpuid_to_vaffinity(v->vcpu_id);
v->arch.actlr = READ_SYSREG32(ACTLR_EL1);
processor_vcpu_initialise(v);
if ( (rc = vcpu_vgic_init(v)) != 0 )
goto fail;
if ( (rc = vcpu_vtimer_init(v)) != 0 )
goto fail;
return rc;
fail:
vcpu_destroy(v);
return rc;
}
static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
{
current->arch.virt_timer.ctl = READ_SYSREG32(CNTV_CTL_EL0);
WRITE_SYSREG32(current->arch.virt_timer.ctl | CNTx_CTL_MASK, CNTV_CTL_EL0);
vgic_vcpu_inject_irq(current, current->arch.virt_timer.irq, 1);
}
static void ctxt_switch_from(struct vcpu *p)
{
p2m_save_state(p);
/* CP 15 */
p->arch.csselr = READ_SYSREG(CSSELR_EL1);
/* Control Registers */
p->arch.cpacr = READ_SYSREG(CPACR_EL1);
p->arch.contextidr = READ_SYSREG(CONTEXTIDR_EL1);
p->arch.tpidr_el0 = READ_SYSREG(TPIDR_EL0);
p->arch.tpidrro_el0 = READ_SYSREG(TPIDRRO_EL0);
p->arch.tpidr_el1 = READ_SYSREG(TPIDR_EL1);
/* Arch timer */
p->arch.cntkctl = READ_SYSREG32(CNTKCTL_EL1);
virt_timer_save(p);
if ( is_32bit_domain(p->domain) && cpu_has_thumbee )
{
p->arch.teecr = READ_SYSREG32(TEECR32_EL1);
p->arch.teehbr = READ_SYSREG32(TEEHBR32_EL1);
}
#ifdef CONFIG_ARM_32
p->arch.joscr = READ_CP32(JOSCR);
p->arch.jmcr = READ_CP32(JMCR);
#endif
isb();
/* MMU */
p->arch.vbar = READ_SYSREG(VBAR_EL1);
p->arch.ttbcr = READ_SYSREG(TCR_EL1);
p->arch.ttbr0 = READ_SYSREG64(TTBR0_EL1);
p->arch.ttbr1 = READ_SYSREG64(TTBR1_EL1);
if ( is_32bit_domain(p->domain) )
p->arch.dacr = READ_SYSREG(DACR32_EL2);
p->arch.par = READ_SYSREG64(PAR_EL1);
#if defined(CONFIG_ARM_32)
p->arch.mair0 = READ_CP32(MAIR0);
p->arch.mair1 = READ_CP32(MAIR1);
p->arch.amair0 = READ_CP32(AMAIR0);
p->arch.amair1 = READ_CP32(AMAIR1);
#else
p->arch.mair = READ_SYSREG64(MAIR_EL1);
p->arch.amair = READ_SYSREG64(AMAIR_EL1);
#endif
/* Fault Status */
#if defined(CONFIG_ARM_32)
p->arch.dfar = READ_CP32(DFAR);
p->arch.ifar = READ_CP32(IFAR);
p->arch.dfsr = READ_CP32(DFSR);
#elif defined(CONFIG_ARM_64)
p->arch.far = READ_SYSREG64(FAR_EL1);
p->arch.esr = READ_SYSREG64(ESR_EL1);
#endif
if ( is_32bit_domain(p->domain) )
p->arch.ifsr = READ_SYSREG(IFSR32_EL2);
p->arch.afsr0 = READ_SYSREG(AFSR0_EL1);
p->arch.afsr1 = READ_SYSREG(AFSR1_EL1);
/* XXX MPU */
/* VFP */
vfp_save_state(p);
/* VGIC */
gic_save_state(p);
isb();
context_saved(p);
}
/* MMU setup for secondary CPUS (which already have paging enabled) */
void __cpuinit mmu_init_secondary_cpu(void)
{
/* From now on, no mapping may be both writable and executable. */
WRITE_SYSREG32(READ_SYSREG32(SCTLR_EL2) | SCTLR_WXN, SCTLR_EL2);
flush_xen_text_tlb();
}
/* Boot-time pagetable setup.
* Changes here may need matching changes in head.S */
void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
{
unsigned long dest_va;
lpae_t pte, *p;
int i;
/* Map the destination in the boot misc area. */
dest_va = BOOT_MISC_VIRT_START;
pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT);
write_pte(xen_second + second_table_offset(dest_va), pte);
flush_xen_data_tlb_range_va(dest_va, SECOND_SIZE);
/* Calculate virt-to-phys offset for the new location */
phys_offset = xen_paddr - (unsigned long) _start;
/* Copy */
memcpy((void *) dest_va, _start, _end - _start);
/* Beware! Any state we modify between now and the PT switch may be
* discarded when we switch over to the copy. */
/* Update the copy of xen_pgtable to use the new paddrs */
p = (void *) xen_pgtable + dest_va - (unsigned long) _start;
#ifdef CONFIG_ARM_64
p[0].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
p = (void *) xen_first + dest_va - (unsigned long) _start;
#endif
for ( i = 0; i < 4; i++)
p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
p = (void *) xen_second + dest_va - (unsigned long) _start;
if ( boot_phys_offset != 0 )
{
/* Remove the old identity mapping of the boot paddr */
vaddr_t va = (vaddr_t)_start + boot_phys_offset;
p[second_linear_offset(va)].bits = 0;
}
for ( i = 0; i < 4 * LPAE_ENTRIES; i++)
if ( p[i].pt.valid )
p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
/* Change pagetables to the copy in the relocated Xen */
boot_ttbr = (uintptr_t) xen_pgtable + phys_offset;
flush_xen_dcache(boot_ttbr);
flush_xen_dcache_va_range((void*)dest_va, _end - _start);
flush_xen_text_tlb();
WRITE_SYSREG64(boot_ttbr, TTBR0_EL2);
dsb(); /* Ensure visibility of HTTBR update */
flush_xen_text_tlb();
/* Undo the temporary map */
pte.bits = 0;
write_pte(xen_second + second_table_offset(dest_va), pte);
flush_xen_text_tlb();
/* Link in the fixmap pagetable */
pte = mfn_to_xen_entry((((unsigned long) xen_fixmap) + phys_offset)
>> PAGE_SHIFT);
pte.pt.table = 1;
write_pte(xen_second + second_table_offset(FIXMAP_ADDR(0)), pte);
/*
* No flush required here. Individual flushes are done in
* set_fixmap as entries are used.
*/
/* Break up the Xen mapping into 4k pages and protect them separately. */
for ( i = 0; i < LPAE_ENTRIES; i++ )
{
unsigned long mfn = paddr_to_pfn(xen_paddr) + i;
unsigned long va = XEN_VIRT_START + (i << PAGE_SHIFT);
if ( !is_kernel(va) )
break;
pte = mfn_to_xen_entry(mfn);
pte.pt.table = 1; /* 4k mappings always have this bit set */
if ( is_kernel_text(va) || is_kernel_inittext(va) )
{
pte.pt.xn = 0;
pte.pt.ro = 1;
}
if ( is_kernel_rodata(va) )
pte.pt.ro = 1;
write_pte(xen_xenmap + i, pte);
/* No flush required here as page table is not hooked in yet. */
}
pte = mfn_to_xen_entry((((unsigned long) xen_xenmap) + phys_offset)
>> PAGE_SHIFT);
pte.pt.table = 1;
write_pte(xen_second + second_linear_offset(XEN_VIRT_START), pte);
/* TLBFLUSH and ISB would be needed here, but wait until we set WXN */
/* From now on, no mapping may be both writable and executable. */
WRITE_SYSREG32(READ_SYSREG32(SCTLR_EL2) | SCTLR_WXN, SCTLR_EL2);
/* Flush everything after setting WXN bit. */
flush_xen_text_tlb();
}