static __init void build_update_entries(u32 **p, unsigned int tmp,
unsigned int ptep)
{
/*
* 64bit address support (36bit on a 32bit CPU) in a 32bit
* Kernel is a special case. Only a few CPUs use it.
*/
#ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits) {
uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
} else {
int pte_off_even = sizeof(pte_t) / 2;
int pte_off_odd = pte_off_even + sizeof(pte_t);
/* The pte entries are pre-shifted */
uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
}
#else
UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
if (r45k_bvahwbug())
build_tlb_probe_entry(p);
UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
if (r4k_250MHZhwbug())
uasm_i_mtc0(p, 0, C0_ENTRYLO0);
uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
if (r45k_bvahwbug())
uasm_i_mfc0(p, tmp, C0_INDEX);
if (r4k_250MHZhwbug())
uasm_i_mtc0(p, 0, C0_ENTRYLO1);
uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
#endif
}
static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
{
/*
* Bug workaround for the Nevada. It seems as if under certain
* circumstances the move from cp0_context might produce a
* bogus result when the mfc0 instruction and its consumer are
* in a different cacheline or a load instruction, probably any
* memory reference, is between them.
*/
switch (current_cpu_type()) {
case CPU_NEVADA:
UASM_i_LW(p, ptr, 0, ptr);
GET_CONTEXT(p, tmp); /* get context reg */
break;
default:
GET_CONTEXT(p, tmp); /* get context reg */
UASM_i_LW(p, ptr, 0, ptr);
break;
}
build_adjust_context(p, tmp);
UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
}
static void __cpuinit
iPTE_LW(u32 **p, struct uasm_label **l, unsigned int pte, unsigned int ptr)
{
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits)
uasm_i_lld(p, pte, 0, ptr);
else
# endif
UASM_i_LL(p, pte, 0, ptr);
#else
# ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits)
uasm_i_ld(p, pte, 0, ptr);
else
# endif
UASM_i_LW(p, pte, 0, ptr);
#endif
}
static void build_bounce_code(unsigned long *spp, unsigned long *gpp)
{
int i;
unsigned long base[32] = {0,};
unsigned int pflag = (unsigned int)KSEG1ADDR(&smp_flag);
unsigned int entry = (unsigned int)__jzsoc_secondary_start;
unsigned int *p;
for(i=0;i<32;i++) {
base[i] = __get_free_pages(GFP_KERNEL, 0);
if(!base[i] || (base[i] & 0xffff))
continue;
smp_bounce.base = base[i];
break;
}
for(i=i-1;i>=0;i--) {
free_pages(base[i], 0);
}
BUG_ON(!smp_bounce.base || (smp_bounce.base & 0xffff));
p = (unsigned int*)smp_bounce.base;
UASM_i_LA(&p, 26, pflag);
UASM_i_LW(&p, 2, 0, 26);
UASM_i_ADDIU(&p, 2, 2, 1);
UASM_i_SW(&p, 2, 0, 26);
/* t7: cpu_start. t8: cpu_ready. t9: cpu_running. */
UASM_i_LA(&p, 15, (unsigned long)cpu_start.bits);
UASM_i_LA(&p, 24, (unsigned long)cpu_ready_e.bits);
UASM_i_LA(&p, 25, (unsigned long)cpu_running.bits);
UASM_i_LA(&p, 29, (unsigned long)spp);
UASM_i_LA(&p, 28, (unsigned long)gpp);
UASM_i_LA(&p, 31, entry);
uasm_i_jr(&p, 31);
uasm_i_nop(&p);
}
/*
* R4000 style TLB load/store/modify handlers.
*/
static void __cpuinit
build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
#ifdef CONFIG_64BIT
build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
#else
build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
#endif
UASM_i_MFC0(p, pte, C0_BADVADDR);
UASM_i_LW(p, ptr, 0, ptr);
UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
UASM_i_ADDU(p, ptr, ptr, pte);
#ifdef CONFIG_SMP
uasm_l_smp_pgtable_change(l, *p);
#endif
iPTE_LW(p, l, pte, ptr); /* get even pte */
if (!m4kc_tlbp_war())
build_tlb_probe_entry(p);
}
/**
* kvm_mips_build_exit() - Assemble common guest exit handler.
* @addr: Address to start writing code.
*
* Assemble the generic guest exit handling code. This is called by the
* exception vectors (generated by kvm_mips_build_exception()), and calls
* kvm_mips_handle_exit(), then either resumes the guest or returns to the host
* depending on the return value.
*
* Returns: Next address after end of written function.
*/
void *kvm_mips_build_exit(void *addr)
{
u32 *p = addr;
unsigned int i;
struct uasm_label labels[3];
struct uasm_reloc relocs[3];
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
/*
* Generic Guest exception handler. We end up here when the guest
* does something that causes a trap to kernel mode.
*
* Both k0/k1 registers will have already been saved (k0 into the vcpu
* structure, and k1 into the scratch_tmp register).
*
* The k1 register will already contain the kvm_vcpu_arch pointer.
*/
/* Start saving Guest context to VCPU */
for (i = 0; i < 32; ++i) {
/* Guest k0/k1 saved later */
if (i == K0 || i == K1)
continue;
UASM_i_SW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
}
#ifndef CONFIG_CPU_MIPSR6
/* We need to save hi/lo and restore them on the way out */
uasm_i_mfhi(&p, T0);
UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, hi), K1);
uasm_i_mflo(&p, T0);
UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, lo), K1);
#endif
/* Finally save guest k1 to VCPU */
uasm_i_ehb(&p);
UASM_i_MFC0(&p, T0, scratch_tmp[0], scratch_tmp[1]);
UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);
/* Now that context has been saved, we can use other registers */
/* Restore vcpu */
UASM_i_MFC0(&p, A1, scratch_vcpu[0], scratch_vcpu[1]);
uasm_i_move(&p, S1, A1);
/* Restore run (vcpu->run) */
UASM_i_LW(&p, A0, offsetof(struct kvm_vcpu, run), A1);
/* Save pointer to run in s0, will be saved by the compiler */
uasm_i_move(&p, S0, A0);
/*
* Save Host level EPC, BadVaddr and Cause to VCPU, useful to process
* the exception
*/
UASM_i_MFC0(&p, K0, C0_EPC);
UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, pc), K1);
UASM_i_MFC0(&p, K0, C0_BADVADDR);
UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_badvaddr),
K1);
uasm_i_mfc0(&p, K0, C0_CAUSE);
uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_cause), K1);
/* Now restore the host state just enough to run the handlers */
/* Switch EBASE to the one used by Linux */
/* load up the host EBASE */
uasm_i_mfc0(&p, V0, C0_STATUS);
uasm_i_lui(&p, AT, ST0_BEV >> 16);
uasm_i_or(&p, K0, V0, AT);
uasm_i_mtc0(&p, K0, C0_STATUS);
uasm_i_ehb(&p);
UASM_i_LA_mostly(&p, K0, (long)&ebase);
UASM_i_LW(&p, K0, uasm_rel_lo((long)&ebase), K0);
build_set_exc_base(&p, K0);
if (raw_cpu_has_fpu) {
/*
* If FPU is enabled, save FCR31 and clear it so that later
* ctc1's don't trigger FPE for pending exceptions.
*/
uasm_i_lui(&p, AT, ST0_CU1 >> 16);
uasm_i_and(&p, V1, V0, AT);
uasm_il_beqz(&p, &r, V1, label_fpu_1);
uasm_i_nop(&p);
uasm_i_cfc1(&p, T0, 31);
uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.fcr31),
K1);
uasm_i_ctc1(&p, ZERO, 31);
uasm_l_fpu_1(&l, p);
}
if (cpu_has_msa) {
/*
* If MSA is enabled, save MSACSR and clear it so that later
* instructions don't trigger MSAFPE for pending exceptions.
*/
uasm_i_mfc0(&p, T0, C0_CONFIG5);
uasm_i_ext(&p, T0, T0, 27, 1); /* MIPS_CONF5_MSAEN */
uasm_il_beqz(&p, &r, T0, label_msa_1);
uasm_i_nop(&p);
uasm_i_cfcmsa(&p, T0, MSA_CSR);
uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.msacsr),
K1);
uasm_i_ctcmsa(&p, MSA_CSR, ZERO);
uasm_l_msa_1(&l, p);
}
/* Now that the new EBASE has been loaded, unset BEV and KSU_USER */
uasm_i_addiu(&p, AT, ZERO, ~(ST0_EXL | KSU_USER | ST0_IE));
uasm_i_and(&p, V0, V0, AT);
uasm_i_lui(&p, AT, ST0_CU0 >> 16);
uasm_i_or(&p, V0, V0, AT);
uasm_i_mtc0(&p, V0, C0_STATUS);
uasm_i_ehb(&p);
/* Load up host GP */
UASM_i_LW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1);
/* Need a stack before we can jump to "C" */
UASM_i_LW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1);
/* Saved host state */
UASM_i_ADDIU(&p, SP, SP, -(int)sizeof(struct pt_regs));
/*
* XXXKYMA do we need to load the host ASID, maybe not because the
* kernel entries are marked GLOBAL, need to verify
*/
/* Restore host scratch registers, as we'll have clobbered them */
kvm_mips_build_restore_scratch(&p, K0, SP);
/* Restore RDHWR access */
UASM_i_LA_mostly(&p, K0, (long)&hwrena);
uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0);
uasm_i_mtc0(&p, K0, C0_HWRENA);
/* Jump to handler */
/*
* XXXKYMA: not sure if this is safe, how large is the stack??
* Now jump to the kvm_mips_handle_exit() to see if we can deal
* with this in the kernel
*/
UASM_i_LA(&p, T9, (unsigned long)kvm_mips_handle_exit);
uasm_i_jalr(&p, RA, T9);
UASM_i_ADDIU(&p, SP, SP, -CALLFRAME_SIZ);
uasm_resolve_relocs(relocs, labels);
p = kvm_mips_build_ret_from_exit(p);
return p;
}
/**
* kvm_mips_build_enter_guest() - Assemble code to resume guest execution.
* @addr: Address to start writing code.
*
* Assemble the code to resume guest execution. This code is common between the
* initial entry into the guest from the host, and returning from the exit
* handler back to the guest.
*
* Returns: Next address after end of written function.
*/
static void *kvm_mips_build_enter_guest(void *addr)
{
u32 *p = addr;
unsigned int i;
struct uasm_label labels[2];
struct uasm_reloc relocs[2];
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
/* Set Guest EPC */
UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, pc), K1);
UASM_i_MTC0(&p, T0, C0_EPC);
/* Set the ASID for the Guest Kernel */
UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, cop0), K1);
UASM_i_LW(&p, T0, offsetof(struct mips_coproc, reg[MIPS_CP0_STATUS][0]),
T0);
uasm_i_andi(&p, T0, T0, KSU_USER | ST0_ERL | ST0_EXL);
uasm_i_xori(&p, T0, T0, KSU_USER);
uasm_il_bnez(&p, &r, T0, label_kernel_asid);
UASM_i_ADDIU(&p, T1, K1,
offsetof(struct kvm_vcpu_arch, guest_kernel_asid));
/* else user */
UASM_i_ADDIU(&p, T1, K1,
offsetof(struct kvm_vcpu_arch, guest_user_asid));
uasm_l_kernel_asid(&l, p);
/* t1: contains the base of the ASID array, need to get the cpu id */
/* smp_processor_id */
uasm_i_lw(&p, T2, offsetof(struct thread_info, cpu), GP);
/* x4 */
uasm_i_sll(&p, T2, T2, 2);
UASM_i_ADDU(&p, T3, T1, T2);
uasm_i_lw(&p, K0, 0, T3);
#ifdef CONFIG_MIPS_ASID_BITS_VARIABLE
/* x sizeof(struct cpuinfo_mips)/4 */
uasm_i_addiu(&p, T3, ZERO, sizeof(struct cpuinfo_mips)/4);
uasm_i_mul(&p, T2, T2, T3);
UASM_i_LA_mostly(&p, AT, (long)&cpu_data[0].asid_mask);
UASM_i_ADDU(&p, AT, AT, T2);
UASM_i_LW(&p, T2, uasm_rel_lo((long)&cpu_data[0].asid_mask), AT);
uasm_i_and(&p, K0, K0, T2);
#else
uasm_i_andi(&p, K0, K0, MIPS_ENTRYHI_ASID);
#endif
uasm_i_mtc0(&p, K0, C0_ENTRYHI);
uasm_i_ehb(&p);
/* Disable RDHWR access */
uasm_i_mtc0(&p, ZERO, C0_HWRENA);
/* load the guest context from VCPU and return */
for (i = 1; i < 32; ++i) {
/* Guest k0/k1 loaded later */
if (i == K0 || i == K1)
continue;
UASM_i_LW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
}
#ifndef CONFIG_CPU_MIPSR6
/* Restore hi/lo */
UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, hi), K1);
uasm_i_mthi(&p, K0);
UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, lo), K1);
uasm_i_mtlo(&p, K0);
#endif
/* Restore the guest's k0/k1 registers */
UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1);
UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);
/* Jump to guest */
uasm_i_eret(&p);
uasm_resolve_relocs(relocs, labels);
return p;
}
