static void kvm_mips_build_save_scratch(u32 **p, unsigned int tmp, unsigned int frame) { /* Save the VCPU scratch register value in cp0_epc of the stack frame */ UASM_i_MFC0(p, tmp, scratch_vcpu[0], scratch_vcpu[1]); UASM_i_SW(p, tmp, offsetof(struct pt_regs, cp0_epc), frame); /* Save the temp scratch register value in cp0_cause of stack frame */ if (scratch_tmp[0] == 31) { UASM_i_MFC0(p, tmp, scratch_tmp[0], scratch_tmp[1]); UASM_i_SW(p, tmp, offsetof(struct pt_regs, cp0_cause), frame); } }
/** * kvm_mips_build_exception() - Assemble first level guest exception handler. * @addr: Address to start writing code. * @handler: Address of common handler (within range of @addr). * * Assemble exception vector code for guest execution. The generated vector will * branch to the common exception handler generated by kvm_mips_build_exit(). * * Returns: Next address after end of written function. */ void *kvm_mips_build_exception(void *addr, void *handler) { u32 *p = addr; struct uasm_label labels[2]; struct uasm_reloc relocs[2]; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); /* Save guest k1 into scratch register */ UASM_i_MTC0(&p, K1, scratch_tmp[0], scratch_tmp[1]); /* Get the VCPU pointer from the VCPU scratch register */ UASM_i_MFC0(&p, K1, scratch_vcpu[0], scratch_vcpu[1]); UASM_i_ADDIU(&p, K1, K1, offsetof(struct kvm_vcpu, arch)); /* Save guest k0 into VCPU structure */ UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1); /* Branch to the common handler */ uasm_il_b(&p, &r, label_exit_common); uasm_i_nop(&p); uasm_l_exit_common(&l, handler); uasm_resolve_relocs(relocs, labels); return p; }
static void __cpuinit build_r4000_tlb_load_handler(void) { u32 *p = handle_tlbl; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(handle_tlbl, 0, sizeof(handle_tlbl)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); if (bcm1250_m3_war()) { UASM_i_MFC0(&p, K0, C0_BADVADDR); UASM_i_MFC0(&p, K1, C0_ENTRYHI); uasm_i_xor(&p, K0, K0, K1); UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); uasm_il_bnez(&p, &r, K0, label_leave); /* No need for uasm_i_nop */ } build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); if (m4kc_tlbp_war()) build_tlb_probe_entry(&p); build_make_valid(&p, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); uasm_l_nopage_tlbl(&l, p); uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); uasm_i_nop(&p); if ((p - handle_tlbl) > FASTPATH_SIZE) panic("TLB load handler fastpath space exceeded"); uasm_resolve_relocs(relocs, labels); pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbl)); dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); }
/* * R4000 style TLB load/store/modify handlers. */ static void __cpuinit build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int pte, unsigned int ptr) { #ifdef CONFIG_64BIT build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ #else build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ #endif UASM_i_MFC0(p, pte, C0_BADVADDR); UASM_i_LW(p, ptr, 0, ptr); UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2); UASM_i_ADDU(p, ptr, ptr, pte); #ifdef CONFIG_SMP uasm_l_smp_pgtable_change(l, *p); #endif iPTE_LW(p, l, pte, ptr); /* get even pte */ if (!m4kc_tlbp_war()) build_tlb_probe_entry(p); }
/** * kvm_mips_build_exit() - Assemble common guest exit handler. * @addr: Address to start writing code. * * Assemble the generic guest exit handling code. This is called by the * exception vectors (generated by kvm_mips_build_exception()), and calls * kvm_mips_handle_exit(), then either resumes the guest or returns to the host * depending on the return value. * * Returns: Next address after end of written function. */ void *kvm_mips_build_exit(void *addr) { u32 *p = addr; unsigned int i; struct uasm_label labels[3]; struct uasm_reloc relocs[3]; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); /* * Generic Guest exception handler. We end up here when the guest * does something that causes a trap to kernel mode. * * Both k0/k1 registers will have already been saved (k0 into the vcpu * structure, and k1 into the scratch_tmp register). * * The k1 register will already contain the kvm_vcpu_arch pointer. */ /* Start saving Guest context to VCPU */ for (i = 0; i < 32; ++i) { /* Guest k0/k1 saved later */ if (i == K0 || i == K1) continue; UASM_i_SW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1); } #ifndef CONFIG_CPU_MIPSR6 /* We need to save hi/lo and restore them on the way out */ uasm_i_mfhi(&p, T0); UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, hi), K1); uasm_i_mflo(&p, T0); UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, lo), K1); #endif /* Finally save guest k1 to VCPU */ uasm_i_ehb(&p); UASM_i_MFC0(&p, T0, scratch_tmp[0], scratch_tmp[1]); UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1); /* Now that context has been saved, we can use other registers */ /* Restore vcpu */ UASM_i_MFC0(&p, A1, scratch_vcpu[0], scratch_vcpu[1]); uasm_i_move(&p, S1, A1); /* Restore run (vcpu->run) */ UASM_i_LW(&p, A0, offsetof(struct kvm_vcpu, run), A1); /* Save pointer to run in s0, will be saved by the compiler */ uasm_i_move(&p, S0, A0); /* * Save Host level EPC, BadVaddr and Cause to VCPU, useful to process * the exception */ UASM_i_MFC0(&p, K0, C0_EPC); UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, pc), K1); UASM_i_MFC0(&p, K0, C0_BADVADDR); UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_badvaddr), K1); uasm_i_mfc0(&p, K0, C0_CAUSE); uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_cause), K1); /* Now restore the host state just enough to run the handlers */ /* Switch EBASE to the one used by Linux */ /* load up the host EBASE */ uasm_i_mfc0(&p, V0, C0_STATUS); uasm_i_lui(&p, AT, ST0_BEV >> 16); uasm_i_or(&p, K0, V0, AT); uasm_i_mtc0(&p, K0, C0_STATUS); uasm_i_ehb(&p); UASM_i_LA_mostly(&p, K0, (long)&ebase); UASM_i_LW(&p, K0, uasm_rel_lo((long)&ebase), K0); build_set_exc_base(&p, K0); if (raw_cpu_has_fpu) { /* * If FPU is enabled, save FCR31 and clear it so that later * ctc1's don't trigger FPE for pending exceptions. */ uasm_i_lui(&p, AT, ST0_CU1 >> 16); uasm_i_and(&p, V1, V0, AT); uasm_il_beqz(&p, &r, V1, label_fpu_1); uasm_i_nop(&p); uasm_i_cfc1(&p, T0, 31); uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.fcr31), K1); uasm_i_ctc1(&p, ZERO, 31); uasm_l_fpu_1(&l, p); } if (cpu_has_msa) { /* * If MSA is enabled, save MSACSR and clear it so that later * instructions don't trigger MSAFPE for pending exceptions. */ uasm_i_mfc0(&p, T0, C0_CONFIG5); uasm_i_ext(&p, T0, T0, 27, 1); /* MIPS_CONF5_MSAEN */ uasm_il_beqz(&p, &r, T0, label_msa_1); uasm_i_nop(&p); uasm_i_cfcmsa(&p, T0, MSA_CSR); uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.msacsr), K1); uasm_i_ctcmsa(&p, MSA_CSR, ZERO); uasm_l_msa_1(&l, p); } /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */ uasm_i_addiu(&p, AT, ZERO, ~(ST0_EXL | KSU_USER | ST0_IE)); uasm_i_and(&p, V0, V0, AT); uasm_i_lui(&p, AT, ST0_CU0 >> 16); uasm_i_or(&p, V0, V0, AT); uasm_i_mtc0(&p, V0, C0_STATUS); uasm_i_ehb(&p); /* Load up host GP */ UASM_i_LW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1); /* Need a stack before we can jump to "C" */ UASM_i_LW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1); /* Saved host state */ UASM_i_ADDIU(&p, SP, SP, -(int)sizeof(struct pt_regs)); /* * XXXKYMA do we need to load the host ASID, maybe not because the * kernel entries are marked GLOBAL, need to verify */ /* Restore host scratch registers, as we'll have clobbered them */ kvm_mips_build_restore_scratch(&p, K0, SP); /* Restore RDHWR access */ UASM_i_LA_mostly(&p, K0, (long)&hwrena); uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0); uasm_i_mtc0(&p, K0, C0_HWRENA); /* Jump to handler */ /* * XXXKYMA: not sure if this is safe, how large is the stack?? * Now jump to the kvm_mips_handle_exit() to see if we can deal * with this in the kernel */ UASM_i_LA(&p, T9, (unsigned long)kvm_mips_handle_exit); uasm_i_jalr(&p, RA, T9); UASM_i_ADDIU(&p, SP, SP, -CALLFRAME_SIZ); uasm_resolve_relocs(relocs, labels); p = kvm_mips_build_ret_from_exit(p); return p; }
static void __cpuinit build_r4000_tlb_refill_handler(void) { u32 *p = tlb_handler; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; u32 *f; unsigned int final_len; int i; memset(tlb_handler, 0, sizeof(tlb_handler)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); memset(final_handler, 0, sizeof(final_handler)); /* * create the plain linear handler */ if (bcm1250_m3_war()) { UASM_i_MFC0(&p, K0, C0_BADVADDR); UASM_i_MFC0(&p, K1, C0_ENTRYHI); uasm_i_xor(&p, K0, K0, K1); UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); uasm_il_bnez(&p, &r, K0, label_leave); /* No need for uasm_i_nop */ } #ifdef CONFIG_64BIT build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ #else build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ #endif build_get_ptep(&p, K0, K1); build_update_entries(&p, K0, K1); build_tlb_write_entry(&p, &l, &r, tlb_random); uasm_l_leave(&l, p); uasm_i_eret(&p); /* return from trap */ #ifdef CONFIG_64BIT build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); #endif /* * Overflow check: For the 64bit handler, we need at least one * free instruction slot for the wrap-around branch. In worst * case, if the intended insertion point is a delay slot, we * need three, with the second nop'ed and the third being * unused. */ /* Loongson2 ebase is different than r4k, we have more space */ #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) if ((p - tlb_handler) > 64) panic("TLB refill handler space exceeded"); #else if (((p - tlb_handler) > 63) || (((p - tlb_handler) > 61) && uasm_insn_has_bdelay(relocs, tlb_handler + 29))) panic("TLB refill handler space exceeded"); #endif /* * Now fold the handler in the TLB refill handler space. */ #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) f = final_handler; /* Simplest case, just copy the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; #else /* CONFIG_64BIT */ f = final_handler + 32; if ((p - tlb_handler) <= 32) { /* Just copy the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; } else { u32 *split = tlb_handler + 30; /* * Find the split point. */ if (uasm_insn_has_bdelay(relocs, split - 1)) split--; /* Copy first part of the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, split, f); f += split - tlb_handler; /* Insert branch. */ uasm_l_split(&l, final_handler); uasm_il_b(&f, &r, label_split); if (uasm_insn_has_bdelay(relocs, split)) uasm_i_nop(&f); else { uasm_copy_handler(relocs, labels, split, split + 1, f); uasm_move_labels(labels, f, f + 1, -1); f++; split++; } /* Copy the rest of the handler. */ uasm_copy_handler(relocs, labels, split, p, final_handler); final_len = (f - (final_handler + 32)) + (p - split); } #endif /* CONFIG_64BIT */ uasm_resolve_relocs(relocs, labels); pr_debug("Wrote TLB refill handler (%u instructions).\n", final_len); f = final_handler; #if defined(CONFIG_64BIT) && !defined(CONFIG_CPU_LOONGSON2) if (final_len > 32) final_len = 64; else f = final_handler + 32; #endif /* CONFIG_64BIT */ pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < final_len; i++) pr_debug("\t.word 0x%08x\n", f[i]); pr_debug("\t.set pop\n"); memcpy((void *)ebase, final_handler, 0x100); }