static void __cpuinit build_r3000_tlb_modify_handler(void) { u32 *p = handle_tlbm; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(handle_tlbm, 0, sizeof(handle_tlbm)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); build_r3000_tlbchange_handler_head(&p, K0, K1); build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); uasm_i_nop(&p); /* load delay */ build_make_write(&p, &r, K0, K1); build_r3000_pte_reload_tlbwi(&p, K0, K1); uasm_l_nopage_tlbm(&l, p); uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); if ((p - handle_tlbm) > FASTPATH_SIZE) panic("TLB modify handler fastpath space exceeded"); uasm_resolve_relocs(relocs, labels); pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbm)); dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); }
static inline void build_copy_store_pref(u32 **buf, int off) { if (off & cache_line_mask()) return; if (pref_bias_copy_store) { uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off, A0); } else if (cache_line_size == (half_copy_loop_size << 1)) { if (cpu_has_cache_cdex_s) { uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); } else if (cpu_has_cache_cdex_p) { if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { uasm_i_nop(buf); uasm_i_nop(buf); uasm_i_nop(buf); uasm_i_nop(buf); } if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) uasm_i_lw(buf, ZERO, ZERO, AT); uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); } } }
static void __cpuinit iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, unsigned int mode) { #ifdef CONFIG_64BIT_PHYS_ADDR unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); #endif uasm_i_ori(p, pte, pte, mode); #ifdef CONFIG_SMP # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) uasm_i_scd(p, pte, 0, ptr); else # endif UASM_i_SC(p, pte, 0, ptr); if (r10000_llsc_war()) uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); else uasm_il_beqz(p, r, pte, label_smp_pgtable_change); # ifdef CONFIG_64BIT_PHYS_ADDR if (!cpu_has_64bits) { /* no uasm_i_nop needed */ uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); uasm_i_ori(p, pte, pte, hwmode); uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); uasm_il_beqz(p, r, pte, label_smp_pgtable_change); /* no uasm_i_nop needed */ uasm_i_lw(p, pte, 0, ptr); } else uasm_i_nop(p); # else uasm_i_nop(p); # endif #else # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) uasm_i_sd(p, pte, 0, ptr); else # endif UASM_i_SW(p, pte, 0, ptr); # ifdef CONFIG_64BIT_PHYS_ADDR if (!cpu_has_64bits) { uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); uasm_i_ori(p, pte, pte, hwmode); uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); uasm_i_lw(p, pte, 0, ptr); } # endif #endif }
static void __cpuinit build_r4000_tlb_modify_handler(void) { u32 *p = handle_tlbm; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(handle_tlbm, 0, sizeof(handle_tlbm)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); if (m4kc_tlbp_war()) build_tlb_probe_entry(&p); /* Present and writable bits set, set accessed and dirty bits. */ build_make_write(&p, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); uasm_l_nopage_tlbm(&l, p); uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); if ((p - handle_tlbm) > FASTPATH_SIZE) panic("TLB modify handler fastpath space exceeded"); uasm_resolve_relocs(relocs, labels); pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbm)); dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); }
/** * kvm_mips_build_exception() - Assemble first level guest exception handler. * @addr: Address to start writing code. * @handler: Address of common handler (within range of @addr). * * Assemble exception vector code for guest execution. The generated vector will * branch to the common exception handler generated by kvm_mips_build_exit(). * * Returns: Next address after end of written function. */ void *kvm_mips_build_exception(void *addr, void *handler) { u32 *p = addr; struct uasm_label labels[2]; struct uasm_reloc relocs[2]; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); /* Save guest k1 into scratch register */ UASM_i_MTC0(&p, K1, scratch_tmp[0], scratch_tmp[1]); /* Get the VCPU pointer from the VCPU scratch register */ UASM_i_MFC0(&p, K1, scratch_vcpu[0], scratch_vcpu[1]); UASM_i_ADDIU(&p, K1, K1, offsetof(struct kvm_vcpu, arch)); /* Save guest k0 into VCPU structure */ UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1); /* Branch to the common handler */ uasm_il_b(&p, &r, label_exit_common); uasm_i_nop(&p); uasm_l_exit_common(&l, handler); uasm_resolve_relocs(relocs, labels); return p; }
/** * kvm_mips_build_ret_to_host() - Assemble code to return to the host. * @addr: Address to start writing code. * * Assemble the code to handle return from the guest exit handler * (kvm_mips_handle_exit()) back to the host, i.e. to the caller of the vcpu_run * function generated by kvm_mips_build_vcpu_run(). * * Returns: Next address after end of written function. */ static void *kvm_mips_build_ret_to_host(void *addr) { u32 *p = addr; unsigned int i; /* EBASE is already pointing to Linux */ UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, host_stack), K1); UASM_i_ADDIU(&p, K1, K1, -(int)sizeof(struct pt_regs)); /* * r2/v0 is the return code, shift it down by 2 (arithmetic) * to recover the err code */ uasm_i_sra(&p, K0, V0, 2); uasm_i_move(&p, V0, K0); /* Load context saved on the host stack */ for (i = 16; i < 31; ++i) { if (i == 24) i = 28; UASM_i_LW(&p, i, offsetof(struct pt_regs, regs[i]), K1); } /* Restore RDHWR access */ UASM_i_LA_mostly(&p, K0, (long)&hwrena); uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0); uasm_i_mtc0(&p, K0, C0_HWRENA); /* Restore RA, which is the address we will return to */ UASM_i_LW(&p, RA, offsetof(struct pt_regs, regs[RA]), K1); uasm_i_jr(&p, RA); uasm_i_nop(&p); return p; }
static void __init cps_gen_set_top_bit(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, unsigned r_addr, int lbl) { uasm_i_lui(pp, t0, uasm_rel_hi(0x80000000)); uasm_build_label(pl, *pp, lbl); uasm_i_ll(pp, t1, 0, r_addr); uasm_i_or(pp, t1, t1, t0); uasm_i_sc(pp, t1, 0, r_addr); uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); }
/* * Hazards * * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: * 2. A timing hazard exists for the TLBP instruction. * * stalling_instruction * TLBP * * The JTLB is being read for the TLBP throughout the stall generated by the * previous instruction. This is not really correct as the stalling instruction * can modify the address used to access the JTLB. The failure symptom is that * the TLBP instruction will use an address created for the stalling instruction * and not the address held in C0_ENHI and thus report the wrong results. * * The software work-around is to not allow the instruction preceding the TLBP * to stall - make it an NOP or some other instruction guaranteed not to stall. * * Errata 2 will not be fixed. This errata is also on the R5000. * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ static __init void __attribute__((unused)) build_tlb_probe_entry(u32 **p) { switch (current_cpu_type()) { /* Found by experiment: R4600 v2.0 needs this, too. */ case CPU_R4600: case CPU_R5000: case CPU_R5000A: case CPU_NEVADA: uasm_i_nop(p); uasm_i_tlbp(p); break; default: uasm_i_tlbp(p); break; } }
/* * Hazards * * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: * 2. A timing hazard exists for the TLBP instruction. * * stalling_instruction * TLBP * * The JTLB is being read for the TLBP throughout the stall generated by the * previous instruction. This is not really correct as the stalling instruction * can modify the address used to access the JTLB. The failure symptom is that * the TLBP instruction will use an address created for the stalling instruction * and not the address held in C0_ENHI and thus report the wrong results. * * The software work-around is to not allow the instruction preceding the TLBP * to stall - make it an NOP or some other instruction guaranteed not to stall. * * Errata 2 will not be fixed. This errata is also on the R5000. * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) { switch (current_cpu_type()) { /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ case CPU_R4600: case CPU_R4700: case CPU_R5000: case CPU_R5000A: case CPU_NEVADA: uasm_i_nop(p); uasm_i_tlbp(p); break; default: uasm_i_tlbp(p); break; } }
static void __init cps_gen_cache_routine(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cache_desc *cache, unsigned op, int lbl) { unsigned cache_size = cache->ways << cache->waybit; unsigned i; const unsigned unroll_lines = 32; /* If the cache isn't present this function has it easy */ if (cache->flags & MIPS_CACHE_NOT_PRESENT) return; /* Load base address */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Calculate end address */ if (cache_size < 0x8000) uasm_i_addiu(pp, t1, t0, cache_size); else UASM_i_LA(pp, t1, (long)(CKSEG0 + cache_size)); /* Start of cache op loop */ uasm_build_label(pl, *pp, lbl); /* Generate the cache ops */ for (i = 0; i < unroll_lines; i++) { if (cpu_has_mips_r6) { uasm_i_cache(pp, op, 0, t0); uasm_i_addiu(pp, t0, t0, cache->linesz); } else { uasm_i_cache(pp, op, i * cache->linesz, t0); } } if (!cpu_has_mips_r6) /* Update the base address */ uasm_i_addiu(pp, t0, t0, unroll_lines * cache->linesz); /* Loop if we haven't reached the end address yet */ uasm_il_bne(pp, pr, t0, t1, lbl); uasm_i_nop(pp); }
/** * kvm_mips_build_ret_from_exit() - Assemble guest exit return handler. * @addr: Address to start writing code. * * Assemble the code to handle the return from kvm_mips_handle_exit(), either * resuming the guest or returning to the host depending on the return value. * * Returns: Next address after end of written function. */ static void *kvm_mips_build_ret_from_exit(void *addr) { u32 *p = addr; struct uasm_label labels[2]; struct uasm_reloc relocs[2]; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); /* Return from handler Make sure interrupts are disabled */ uasm_i_di(&p, ZERO); uasm_i_ehb(&p); /* * XXXKYMA: k0/k1 could have been blown away if we processed * an exception while we were handling the exception from the * guest, reload k1 */ uasm_i_move(&p, K1, S1); UASM_i_ADDIU(&p, K1, K1, offsetof(struct kvm_vcpu, arch)); /* * Check return value, should tell us if we are returning to the * host (handle I/O etc)or resuming the guest */ uasm_i_andi(&p, T0, V0, RESUME_HOST); uasm_il_bnez(&p, &r, T0, label_return_to_host); uasm_i_nop(&p); p = kvm_mips_build_ret_to_guest(p); uasm_l_return_to_host(&l, p); p = kvm_mips_build_ret_to_host(p); uasm_resolve_relocs(relocs, labels); return p; }
/* * The R3000 TLB handler is simple. */ static void __init build_r3000_tlb_refill_handler(void) { long pgdc = (long)pgd_current; u32 *p; int i; memset(tlb_handler, 0, sizeof(tlb_handler)); p = tlb_handler; uasm_i_mfc0(&p, K0, C0_BADVADDR); uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); uasm_i_srl(&p, K0, K0, 22); /* load delay */ uasm_i_sll(&p, K0, K0, 2); uasm_i_addu(&p, K1, K1, K0); uasm_i_mfc0(&p, K0, C0_CONTEXT); uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ uasm_i_addu(&p, K1, K1, K0); uasm_i_lw(&p, K0, 0, K1); uasm_i_nop(&p); /* load delay */ uasm_i_mtc0(&p, K0, C0_ENTRYLO0); uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ uasm_i_tlbwr(&p); /* cp0 delay */ uasm_i_jr(&p, K1); uasm_i_rfe(&p); /* branch delay */ if (p > tlb_handler + 32) panic("TLB refill handler space exceeded"); pr_debug("Wrote TLB refill handler (%u instructions).\n", (unsigned int)(p - tlb_handler)); pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < (p - tlb_handler); i++) pr_debug("\t.word 0x%08x\n", tlb_handler[i]); pr_debug("\t.set pop\n"); memcpy((void *)ebase, tlb_handler, 0x80); }
static void __cpuinit build_r4000_tlb_load_handler(void) { u32 *p = handle_tlbl; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(handle_tlbl, 0, sizeof(handle_tlbl)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); if (bcm1250_m3_war()) { UASM_i_MFC0(&p, K0, C0_BADVADDR); UASM_i_MFC0(&p, K1, C0_ENTRYHI); uasm_i_xor(&p, K0, K0, K1); UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); uasm_il_bnez(&p, &r, K0, label_leave); /* No need for uasm_i_nop */ } build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); if (m4kc_tlbp_war()) build_tlb_probe_entry(&p); build_make_valid(&p, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); uasm_l_nopage_tlbl(&l, p); uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); uasm_i_nop(&p); if ((p - handle_tlbl) > FASTPATH_SIZE) panic("TLB load handler fastpath space exceeded"); uasm_resolve_relocs(relocs, labels); pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbl)); dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); }
static void build_bounce_code(unsigned long *spp, unsigned long *gpp) { int i; unsigned long base[32] = {0,}; unsigned int pflag = (unsigned int)KSEG1ADDR(&smp_flag); unsigned int entry = (unsigned int)__jzsoc_secondary_start; unsigned int *p; for(i=0;i<32;i++) { base[i] = __get_free_pages(GFP_KERNEL, 0); if(!base[i] || (base[i] & 0xffff)) continue; smp_bounce.base = base[i]; break; } for(i=i-1;i>=0;i--) { free_pages(base[i], 0); } BUG_ON(!smp_bounce.base || (smp_bounce.base & 0xffff)); p = (unsigned int*)smp_bounce.base; UASM_i_LA(&p, 26, pflag); UASM_i_LW(&p, 2, 0, 26); UASM_i_ADDIU(&p, 2, 2, 1); UASM_i_SW(&p, 2, 0, 26); /* t7: cpu_start. t8: cpu_ready. t9: cpu_running. */ UASM_i_LA(&p, 15, (unsigned long)cpu_start.bits); UASM_i_LA(&p, 24, (unsigned long)cpu_ready_e.bits); UASM_i_LA(&p, 25, (unsigned long)cpu_running.bits); UASM_i_LA(&p, 29, (unsigned long)spp); UASM_i_LA(&p, 28, (unsigned long)gpp); UASM_i_LA(&p, 31, entry); uasm_i_jr(&p, 31); uasm_i_nop(&p); }
/** * kvm_mips_build_exit() - Assemble common guest exit handler. * @addr: Address to start writing code. * * Assemble the generic guest exit handling code. This is called by the * exception vectors (generated by kvm_mips_build_exception()), and calls * kvm_mips_handle_exit(), then either resumes the guest or returns to the host * depending on the return value. * * Returns: Next address after end of written function. */ void *kvm_mips_build_exit(void *addr) { u32 *p = addr; unsigned int i; struct uasm_label labels[3]; struct uasm_reloc relocs[3]; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); /* * Generic Guest exception handler. We end up here when the guest * does something that causes a trap to kernel mode. * * Both k0/k1 registers will have already been saved (k0 into the vcpu * structure, and k1 into the scratch_tmp register). * * The k1 register will already contain the kvm_vcpu_arch pointer. */ /* Start saving Guest context to VCPU */ for (i = 0; i < 32; ++i) { /* Guest k0/k1 saved later */ if (i == K0 || i == K1) continue; UASM_i_SW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1); } #ifndef CONFIG_CPU_MIPSR6 /* We need to save hi/lo and restore them on the way out */ uasm_i_mfhi(&p, T0); UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, hi), K1); uasm_i_mflo(&p, T0); UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, lo), K1); #endif /* Finally save guest k1 to VCPU */ uasm_i_ehb(&p); UASM_i_MFC0(&p, T0, scratch_tmp[0], scratch_tmp[1]); UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1); /* Now that context has been saved, we can use other registers */ /* Restore vcpu */ UASM_i_MFC0(&p, A1, scratch_vcpu[0], scratch_vcpu[1]); uasm_i_move(&p, S1, A1); /* Restore run (vcpu->run) */ UASM_i_LW(&p, A0, offsetof(struct kvm_vcpu, run), A1); /* Save pointer to run in s0, will be saved by the compiler */ uasm_i_move(&p, S0, A0); /* * Save Host level EPC, BadVaddr and Cause to VCPU, useful to process * the exception */ UASM_i_MFC0(&p, K0, C0_EPC); UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, pc), K1); UASM_i_MFC0(&p, K0, C0_BADVADDR); UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_badvaddr), K1); uasm_i_mfc0(&p, K0, C0_CAUSE); uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_cause), K1); /* Now restore the host state just enough to run the handlers */ /* Switch EBASE to the one used by Linux */ /* load up the host EBASE */ uasm_i_mfc0(&p, V0, C0_STATUS); uasm_i_lui(&p, AT, ST0_BEV >> 16); uasm_i_or(&p, K0, V0, AT); uasm_i_mtc0(&p, K0, C0_STATUS); uasm_i_ehb(&p); UASM_i_LA_mostly(&p, K0, (long)&ebase); UASM_i_LW(&p, K0, uasm_rel_lo((long)&ebase), K0); build_set_exc_base(&p, K0); if (raw_cpu_has_fpu) { /* * If FPU is enabled, save FCR31 and clear it so that later * ctc1's don't trigger FPE for pending exceptions. */ uasm_i_lui(&p, AT, ST0_CU1 >> 16); uasm_i_and(&p, V1, V0, AT); uasm_il_beqz(&p, &r, V1, label_fpu_1); uasm_i_nop(&p); uasm_i_cfc1(&p, T0, 31); uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.fcr31), K1); uasm_i_ctc1(&p, ZERO, 31); uasm_l_fpu_1(&l, p); } if (cpu_has_msa) { /* * If MSA is enabled, save MSACSR and clear it so that later * instructions don't trigger MSAFPE for pending exceptions. */ uasm_i_mfc0(&p, T0, C0_CONFIG5); uasm_i_ext(&p, T0, T0, 27, 1); /* MIPS_CONF5_MSAEN */ uasm_il_beqz(&p, &r, T0, label_msa_1); uasm_i_nop(&p); uasm_i_cfcmsa(&p, T0, MSA_CSR); uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.msacsr), K1); uasm_i_ctcmsa(&p, MSA_CSR, ZERO); uasm_l_msa_1(&l, p); } /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */ uasm_i_addiu(&p, AT, ZERO, ~(ST0_EXL | KSU_USER | ST0_IE)); uasm_i_and(&p, V0, V0, AT); uasm_i_lui(&p, AT, ST0_CU0 >> 16); uasm_i_or(&p, V0, V0, AT); uasm_i_mtc0(&p, V0, C0_STATUS); uasm_i_ehb(&p); /* Load up host GP */ UASM_i_LW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1); /* Need a stack before we can jump to "C" */ UASM_i_LW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1); /* Saved host state */ UASM_i_ADDIU(&p, SP, SP, -(int)sizeof(struct pt_regs)); /* * XXXKYMA do we need to load the host ASID, maybe not because the * kernel entries are marked GLOBAL, need to verify */ /* Restore host scratch registers, as we'll have clobbered them */ kvm_mips_build_restore_scratch(&p, K0, SP); /* Restore RDHWR access */ UASM_i_LA_mostly(&p, K0, (long)&hwrena); uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0); uasm_i_mtc0(&p, K0, C0_HWRENA); /* Jump to handler */ /* * XXXKYMA: not sure if this is safe, how large is the stack?? * Now jump to the kvm_mips_handle_exit() to see if we can deal * with this in the kernel */ UASM_i_LA(&p, T9, (unsigned long)kvm_mips_handle_exit); uasm_i_jalr(&p, RA, T9); UASM_i_ADDIU(&p, SP, SP, -CALLFRAME_SIZ); uasm_resolve_relocs(relocs, labels); p = kvm_mips_build_ret_from_exit(p); return p; }
static void * __init cps_gen_entry_code(unsigned cpu, enum cps_pm_state state) { struct uasm_label *l = labels; struct uasm_reloc *r = relocs; u32 *buf, *p; const unsigned r_online = a0; const unsigned r_nc_count = a1; const unsigned r_pcohctl = t7; const unsigned max_instrs = 256; unsigned cpc_cmd; enum { lbl_incready = 1, lbl_poll_cont, lbl_secondary_hang, lbl_disable_coherence, lbl_flush_fsb, lbl_invicache, lbl_flushdcache, lbl_hang, lbl_set_cont, lbl_secondary_cont, lbl_decready, }; /* Allocate a buffer to hold the generated code */ p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL); if (!buf) return NULL; /* Clear labels & relocs ready for (re)use */ memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); if (state == CPS_PM_POWER_GATED) { /* Power gating relies upon CPS SMP */ if (!mips_cps_smp_in_use()) goto out_err; /* * Save CPU state. Note the non-standard calling convention * with the return address placed in v0 to avoid clobbering * the ra register before it is saved. */ UASM_i_LA(&p, t0, (long)mips_cps_pm_save); uasm_i_jalr(&p, v0, t0); uasm_i_nop(&p); } /* * Load addresses of required CM & CPC registers. This is done early * because they're needed in both the enable & disable coherence steps * but in the coupled case the enable step will only run on one VPE. */ UASM_i_LA(&p, r_pcohctl, (long)_gcmp_base + GCMPCLCBOFS(COHCTL)); if (coupled_coherence) { /* Increment ready_count */ uasm_i_sync(&p, stype_ordering); uasm_build_label(&l, p, lbl_incready); uasm_i_ll(&p, t1, 0, r_nc_count); uasm_i_addiu(&p, t2, t1, 1); uasm_i_sc(&p, t2, 0, r_nc_count); uasm_il_beqz(&p, &r, t2, lbl_incready); uasm_i_addiu(&p, t1, t1, 1); /* Ordering barrier */ uasm_i_sync(&p, stype_ordering); /* * If this is the last VPE to become ready for non-coherence * then it should branch below. */ uasm_il_beq(&p, &r, t1, r_online, lbl_disable_coherence); uasm_i_nop(&p); if (state < CPS_PM_POWER_GATED) { /* * Otherwise this is not the last VPE to become ready * for non-coherence. It needs to wait until coherence * has been disabled before proceeding, which it will do * by polling for the top bit of ready_count being set. */ uasm_i_addiu(&p, t1, zero, -1); uasm_build_label(&l, p, lbl_poll_cont); uasm_i_lw(&p, t0, 0, r_nc_count); uasm_il_bltz(&p, &r, t0, lbl_secondary_cont); uasm_i_ehb(&p); uasm_i_yield(&p, zero, t1); uasm_il_b(&p, &r, lbl_poll_cont); uasm_i_nop(&p); } else { /* * The core will lose power & this VPE will not continue * so it can simply halt here. */ uasm_i_addiu(&p, t0, zero, TCHALT_H); uasm_i_mtc0(&p, t0, 2, 4); uasm_build_label(&l, p, lbl_secondary_hang); uasm_il_b(&p, &r, lbl_secondary_hang); uasm_i_nop(&p); } } /* * This is the point of no return - this VPE will now proceed to * disable coherence. At this point we *must* be sure that no other * VPE within the core will interfere with the L1 dcache. */ uasm_build_label(&l, p, lbl_disable_coherence); /* Invalidate the L1 icache */ cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache, Index_Invalidate_I, lbl_invicache); /* Writeback & invalidate the L1 dcache */ cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache, Index_Writeback_Inv_D, lbl_flushdcache); /* Completion barrier */ uasm_i_sync(&p, stype_memory); uasm_i_ehb(&p); /* * Disable all but self interventions. The load from COHCTL is defined * by the interAptiv & proAptiv SUMs as ensuring that the operation * resulting from the preceeding store is complete. */ uasm_i_addiu(&p, t0, zero, 1 << cpu_data[cpu].core); uasm_i_sw(&p, t0, 0, r_pcohctl); uasm_i_lw(&p, t0, 0, r_pcohctl); /* Sync to ensure previous interventions are complete */ uasm_i_sync(&p, stype_intervention); uasm_i_ehb(&p); /* Disable coherence */ uasm_i_sw(&p, zero, 0, r_pcohctl); uasm_i_lw(&p, t0, 0, r_pcohctl); if (state >= CPS_PM_CLOCK_GATED) { /* TODO: determine whether required based on CPC version */ cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu].dcache, lbl_flush_fsb); /* Determine the CPC command to issue */ switch (state) { case CPS_PM_CLOCK_GATED: cpc_cmd = CPC_Cx_CMD_CLOCKOFF; break; case CPS_PM_POWER_GATED: cpc_cmd = CPC_Cx_CMD_PWRDOWN; break; default: BUG(); goto out_err; } /* Issue the CPC command */ UASM_i_LA(&p, t0, (long)addr_cpc_cl_cmd()); uasm_i_addiu(&p, t1, zero, cpc_cmd); uasm_i_sw(&p, t1, 0, t0); if (state == CPS_PM_POWER_GATED) { /* If anything goes wrong just hang */ uasm_build_label(&l, p, lbl_hang); uasm_il_b(&p, &r, lbl_hang); uasm_i_nop(&p); /* * There's no point generating more code, the core is * powered down & if powered back up will run from the * reset vector not from here. */ goto gen_done; } /* Completion barrier */ uasm_i_sync(&p, stype_memory); uasm_i_ehb(&p); } if (state == CPS_PM_NC_WAIT) { /* * At this point it is safe for all VPEs to proceed with * execution. This VPE will set the top bit of ready_count * to indicate to the other VPEs that they may continue. */ if (coupled_coherence) cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); /* * VPEs which did not disable coherence will continue * executing, after coherence has been disabled, from this * point. */ uasm_build_label(&l, p, lbl_secondary_cont); /* Now perform our wait */ uasm_i_wait(&p, 0); } /* * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs * will run this. The first will actually re-enable coherence & the * rest will just be performing a rather unusual nop. */ uasm_i_addiu(&p, t0, zero, GCMP_CCB_COHCTL_DOMAIN_MSK); uasm_i_sw(&p, t0, 0, r_pcohctl); uasm_i_lw(&p, t0, 0, r_pcohctl); /* Completion barrier */ uasm_i_sync(&p, stype_memory); uasm_i_ehb(&p); if (coupled_coherence && (state == CPS_PM_NC_WAIT)) { /* Decrement ready_count */ uasm_build_label(&l, p, lbl_decready); uasm_i_sync(&p, stype_ordering); uasm_i_ll(&p, t1, 0, r_nc_count); uasm_i_addiu(&p, t2, t1, -1); uasm_i_sc(&p, t2, 0, r_nc_count); uasm_il_beqz(&p, &r, t2, lbl_decready); uasm_i_andi(&p, v0, t1, (1 << fls(smp_num_siblings)) - 1); /* Ordering barrier */ uasm_i_sync(&p, stype_ordering); } if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) { /* * At this point it is safe for all VPEs to proceed with * execution. This VPE will set the top bit of ready_count * to indicate to the other VPEs that they may continue. */ cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); /* * This core will be reliant upon another core sending a * power-up command to the CPC in order to resume operation. * Thus an arbitrary VPE can't trigger the core leaving the * idle state and the one that disables coherence might as well * be the one to re-enable it. The rest will continue from here * after that has been done. */ uasm_build_label(&l, p, lbl_secondary_cont); /* Ordering barrier */ uasm_i_sync(&p, stype_ordering); } /* The core is coherent, time to return to C code */ uasm_i_jr(&p, ra); uasm_i_nop(&p); gen_done: /* Ensure the code didn't exceed the resources allocated for it */ BUG_ON((p - buf) > max_instrs); BUG_ON((l - labels) > ARRAY_SIZE(labels)); BUG_ON((r - relocs) > ARRAY_SIZE(relocs)); /* Patch branch offsets */ uasm_resolve_relocs(relocs, labels); /* Flush the icache */ local_flush_icache_range((unsigned long)buf, (unsigned long)p); return buf; out_err: kfree(buf); return NULL; }
static int __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cpuinfo_mips *cpu_info, int lbl) { unsigned i, fsb_size = 8; unsigned num_loads = (fsb_size * 3) / 2; unsigned line_stride = 2; unsigned line_size = cpu_info->dcache.linesz; unsigned perf_counter, perf_event; unsigned revision = cpu_info->processor_id & PRID_REV_MASK; /* * Determine whether this CPU requires an FSB flush, and if so which * performance counter/event reflect stalls due to a full FSB. */ switch (__get_cpu_type(cpu_info->cputype)) { case CPU_INTERAPTIV: perf_counter = 1; perf_event = 51; break; case CPU_PROAPTIV: /* Newer proAptiv cores don't require this workaround */ if (revision >= PRID_REV_ENCODE_332(1, 1, 0)) return 0; /* On older ones it's unavailable */ return -1; /* CPUs which do not require the workaround */ case CPU_P5600: case CPU_I6400: return 0; default: WARN_ONCE(1, "pm-cps: FSB flush unsupported for this CPU\n"); return -1; } /* * Ensure that the fill/store buffer (FSB) is not holding the results * of a prefetch, since if it is then the CPC sequencer may become * stuck in the D3 (ClrBus) state whilst entering a low power state. */ /* Preserve perf counter setup */ uasm_i_mfc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_mfc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ /* Setup perf counter to count FSB full pipeline stalls */ uasm_i_addiu(pp, t0, zero, (perf_event << 5) | 0xf); uasm_i_mtc0(pp, t0, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_ehb(pp); uasm_i_mtc0(pp, zero, 25, (perf_counter * 2) + 1); /* PerfCntN */ uasm_i_ehb(pp); /* Base address for loads */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Start of clear loop */ uasm_build_label(pl, *pp, lbl); /* Perform some loads to fill the FSB */ for (i = 0; i < num_loads; i++) uasm_i_lw(pp, zero, i * line_size * line_stride, t0); /* * Invalidate the new D-cache entries so that the cache will need * refilling (via the FSB) if the loop is executed again. */ for (i = 0; i < num_loads; i++) { uasm_i_cache(pp, Hit_Invalidate_D, i * line_size * line_stride, t0); uasm_i_cache(pp, Hit_Writeback_Inv_SD, i * line_size * line_stride, t0); } /* Completion barrier */ uasm_i_sync(pp, stype_memory); uasm_i_ehb(pp); /* Check whether the pipeline stalled due to the FSB being full */ uasm_i_mfc0(pp, t1, 25, (perf_counter * 2) + 1); /* PerfCntN */ /* Loop if it didn't */ uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); /* Restore perf counter 1. The count may well now be wrong... */ uasm_i_mtc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_ehb(pp); uasm_i_mtc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ uasm_i_ehb(pp); return 0; }
/* * BVADDR is the faulting address, PTR is scratch. * PTR will hold the pgd for vmalloc. */ static __init void build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int bvaddr, unsigned int ptr) { long swpd = (long)swapper_pg_dir; #ifdef MODULE_START long modd = (long)module_pg_dir; uasm_l_module_alloc(l, *p); /* * Assumption: * VMALLOC_START >= 0xc000000000000000UL * MODULE_START >= 0xe000000000000000UL */ UASM_i_SLL(p, ptr, bvaddr, 2); uasm_il_bgez(p, r, ptr, label_vmalloc); if (uasm_in_compat_space_p(MODULE_START) && !uasm_rel_lo(MODULE_START)) { uasm_i_lui(p, ptr, uasm_rel_hi(MODULE_START)); /* delay slot */ } else { /* unlikely configuration */ uasm_i_nop(p); /* delay slot */ UASM_i_LA(p, ptr, MODULE_START); } uasm_i_dsubu(p, bvaddr, bvaddr, ptr); if (uasm_in_compat_space_p(modd) && !uasm_rel_lo(modd)) { uasm_il_b(p, r, label_vmalloc_done); uasm_i_lui(p, ptr, uasm_rel_hi(modd)); } else { UASM_i_LA_mostly(p, ptr, modd); uasm_il_b(p, r, label_vmalloc_done); if (uasm_in_compat_space_p(modd)) uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(modd)); else uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(modd)); } uasm_l_vmalloc(l, *p); if (uasm_in_compat_space_p(MODULE_START) && !uasm_rel_lo(MODULE_START) && MODULE_START << 32 == VMALLOC_START) uasm_i_dsll32(p, ptr, ptr, 0); /* typical case */ else UASM_i_LA(p, ptr, VMALLOC_START); #else uasm_l_vmalloc(l, *p); UASM_i_LA(p, ptr, VMALLOC_START); #endif uasm_i_dsubu(p, bvaddr, bvaddr, ptr); if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { uasm_il_b(p, r, label_vmalloc_done); uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); } else { UASM_i_LA_mostly(p, ptr, swpd); uasm_il_b(p, r, label_vmalloc_done); if (uasm_in_compat_space_p(swpd)) uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); else uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); } }
static void __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cache_desc *dcache, int lbl) { unsigned i, fsb_size = 8; unsigned num_loads = (fsb_size * 3) / 2; unsigned line_stride = 2; /* * Ensure that the fill/store buffer (FSB) is not holding the results * of a prefetch, since if it is then the CPC sequencer may become * stuck in the D3 (ClrBus) state whilst entering a low power state. */ /* TODO: this is interAptiv-specific, generalise it */ /* Preserve perf counter 1 setup */ uasm_i_mfc0(pp, t2, 25, 2); /* PerfCtl1 */ uasm_i_mfc0(pp, t3, 25, 3); /* PerfCnt1 */ /* Setup perf counter 1 to count FSB full pipeline stalls */ uasm_i_addiu(pp, t0, zero, 0x66f); uasm_i_mtc0(pp, t0, 25, 2); /* PerfCtl1 */ uasm_i_ehb(pp); uasm_i_mtc0(pp, zero, 25, 3); /* PerfCnt1 */ uasm_i_ehb(pp); /* Base address for loads */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Start of clear loop */ uasm_build_label(pl, *pp, lbl); /* Perform some loads to fill the FSB */ for (i = 0; i < num_loads; i++) uasm_i_lw(pp, zero, i * dcache->linesz * line_stride, t0); /* * Invalidate the new D-cache entries so that the cache will need * refilling (via the FSB) if the loop is executed again. */ for (i = 0; i < num_loads; i++) { uasm_i_cache(pp, Hit_Invalidate_D, i * dcache->linesz * line_stride, t0); uasm_i_cache(pp, Hit_Writeback_Inv_SD, i * dcache->linesz * line_stride, t0); } /* Completion barrier */ uasm_i_sync(pp, stype_memory); uasm_i_ehb(pp); /* Check whether the pipeline stalled due to the FSB being full */ uasm_i_mfc0(pp, t1, 25, 3); /* PerfCnt1 */ /* Loop if it didn't */ uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); /* Restore perf counter 1. The count may well now be wrong... */ uasm_i_mtc0(pp, t2, 25, 2); /* PerfCtl1 */ uasm_i_ehb(pp); uasm_i_mtc0(pp, t3, 25, 3); /* PerfCnt1 */ uasm_i_ehb(pp); }
static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, struct uasm_reloc **r, enum tlb_write_entry wmode) { void(*tlbw)(u32 **) = NULL; switch (wmode) { case tlb_random: tlbw = uasm_i_tlbwr; break; case tlb_indexed: tlbw = uasm_i_tlbwi; break; } if (cpu_has_mips_r2) { if (cpu_has_mips_r2_exec_hazard) uasm_i_ehb(p); tlbw(p); return; } switch (current_cpu_type()) { case CPU_R4000PC: case CPU_R4000SC: case CPU_R4000MC: case CPU_R4400PC: case CPU_R4400SC: case CPU_R4400MC: /* * This branch uses up a mtc0 hazard nop slot and saves * two nops after the tlbw instruction. */ uasm_il_bgezl(p, r, 0, label_tlbw_hazard); tlbw(p); uasm_l_tlbw_hazard(l, *p); uasm_i_nop(p); break; case CPU_R4600: case CPU_R4700: case CPU_R5000: case CPU_R5000A: uasm_i_nop(p); tlbw(p); uasm_i_nop(p); break; case CPU_R4300: case CPU_5KC: case CPU_TX49XX: case CPU_PR4450: uasm_i_nop(p); tlbw(p); break; case CPU_R10000: case CPU_R12000: case CPU_R14000: case CPU_4KC: case CPU_4KEC: case CPU_SB1: case CPU_SB1A: case CPU_4KSC: case CPU_20KC: case CPU_25KF: case CPU_BCM3302: case CPU_BCM4710: case CPU_LOONGSON2: case CPU_R5500: if (m4kc_tlbp_war()) uasm_i_nop(p); case CPU_ALCHEMY: tlbw(p); break; case CPU_NEVADA: uasm_i_nop(p); /* QED specifies 2 nops hazard */ /* * This branch uses up a mtc0 hazard nop slot and saves * a nop after the tlbw instruction. */ uasm_il_bgezl(p, r, 0, label_tlbw_hazard); tlbw(p); uasm_l_tlbw_hazard(l, *p); break; case CPU_RM7000: uasm_i_nop(p); uasm_i_nop(p); uasm_i_nop(p); uasm_i_nop(p); tlbw(p); break; case CPU_RM9000: /* * When the JTLB is updated by tlbwi or tlbwr, a subsequent * use of the JTLB for instructions should not occur for 4 * cpu cycles and use for data translations should not occur * for 3 cpu cycles. */ uasm_i_ssnop(p); uasm_i_ssnop(p); uasm_i_ssnop(p); uasm_i_ssnop(p); tlbw(p); uasm_i_ssnop(p); uasm_i_ssnop(p); uasm_i_ssnop(p); uasm_i_ssnop(p); break; case CPU_VR4111: case CPU_VR4121: case CPU_VR4122: case CPU_VR4181: case CPU_VR4181A: uasm_i_nop(p); uasm_i_nop(p); tlbw(p); uasm_i_nop(p); uasm_i_nop(p); break; case CPU_VR4131: case CPU_VR4133: case CPU_R5432: uasm_i_nop(p); uasm_i_nop(p); tlbw(p); break; default: panic("No TLB refill handler yet (CPU type: %d)", current_cpu_data.cputype); break; } }
static void __cpuinit build_r4000_tlb_refill_handler(void) { u32 *p = tlb_handler; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; u32 *f; unsigned int final_len; int i; memset(tlb_handler, 0, sizeof(tlb_handler)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); memset(final_handler, 0, sizeof(final_handler)); /* * create the plain linear handler */ if (bcm1250_m3_war()) { UASM_i_MFC0(&p, K0, C0_BADVADDR); UASM_i_MFC0(&p, K1, C0_ENTRYHI); uasm_i_xor(&p, K0, K0, K1); UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); uasm_il_bnez(&p, &r, K0, label_leave); /* No need for uasm_i_nop */ } #ifdef CONFIG_64BIT build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ #else build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ #endif build_get_ptep(&p, K0, K1); build_update_entries(&p, K0, K1); build_tlb_write_entry(&p, &l, &r, tlb_random); uasm_l_leave(&l, p); uasm_i_eret(&p); /* return from trap */ #ifdef CONFIG_64BIT build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); #endif /* * Overflow check: For the 64bit handler, we need at least one * free instruction slot for the wrap-around branch. In worst * case, if the intended insertion point is a delay slot, we * need three, with the second nop'ed and the third being * unused. */ /* Loongson2 ebase is different than r4k, we have more space */ #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) if ((p - tlb_handler) > 64) panic("TLB refill handler space exceeded"); #else if (((p - tlb_handler) > 63) || (((p - tlb_handler) > 61) && uasm_insn_has_bdelay(relocs, tlb_handler + 29))) panic("TLB refill handler space exceeded"); #endif /* * Now fold the handler in the TLB refill handler space. */ #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) f = final_handler; /* Simplest case, just copy the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; #else /* CONFIG_64BIT */ f = final_handler + 32; if ((p - tlb_handler) <= 32) { /* Just copy the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; } else { u32 *split = tlb_handler + 30; /* * Find the split point. */ if (uasm_insn_has_bdelay(relocs, split - 1)) split--; /* Copy first part of the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, split, f); f += split - tlb_handler; /* Insert branch. */ uasm_l_split(&l, final_handler); uasm_il_b(&f, &r, label_split); if (uasm_insn_has_bdelay(relocs, split)) uasm_i_nop(&f); else { uasm_copy_handler(relocs, labels, split, split + 1, f); uasm_move_labels(labels, f, f + 1, -1); f++; split++; } /* Copy the rest of the handler. */ uasm_copy_handler(relocs, labels, split, p, final_handler); final_len = (f - (final_handler + 32)) + (p - split); } #endif /* CONFIG_64BIT */ uasm_resolve_relocs(relocs, labels); pr_debug("Wrote TLB refill handler (%u instructions).\n", final_len); f = final_handler; #if defined(CONFIG_64BIT) && !defined(CONFIG_CPU_LOONGSON2) if (final_len > 32) final_len = 64; else f = final_handler + 32; #endif /* CONFIG_64BIT */ pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < final_len; i++) pr_debug("\t.word 0x%08x\n", f[i]); pr_debug("\t.set pop\n"); memcpy((void *)ebase, final_handler, 0x100); }
void build_copy_page(void) { int off; u32 *buf = (u32 *)©_page_array; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; int i; memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); set_prefetch_parameters(); /* * This algorithm makes the following assumptions: * - All prefetch biases are multiples of 8 words. * - The prefetch biases are less than one page. * - The store prefetch bias isn't greater than the load * prefetch bias. */ BUG_ON(pref_bias_copy_load % (8 * copy_word_size)); BUG_ON(pref_bias_copy_store % (8 * copy_word_size)); BUG_ON(PAGE_SIZE < pref_bias_copy_load); BUG_ON(pref_bias_copy_store > pref_bias_copy_load); off = PAGE_SIZE - pref_bias_copy_load; if (off > 0xffff || !pref_bias_copy_load) pg_addiu(&buf, A2, A0, off); else uasm_i_ori(&buf, A2, A0, off); if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) uasm_i_lui(&buf, AT, 0xa000); off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) * cache_line_size : 0; while (off) { build_copy_load_pref(&buf, -off); off -= cache_line_size; } off = cache_line_size ? min(8, pref_bias_copy_store / cache_line_size) * cache_line_size : 0; while (off) { build_copy_store_pref(&buf, -off); off -= cache_line_size; } uasm_l_copy_pref_both(&l, buf); do { build_copy_load_pref(&buf, off); build_copy_load(&buf, T0, off); build_copy_load_pref(&buf, off + copy_word_size); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load_pref(&buf, off + 2 * copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load_pref(&buf, off + 3 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < half_copy_loop_size); pg_addiu(&buf, A1, A1, 2 * off); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_copy_load_pref(&buf, off); build_copy_load(&buf, T0, off); build_copy_load_pref(&buf, off + copy_word_size); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load_pref(&buf, off + 2 * copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load_pref(&buf, off + 3 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); if (off == -(4 * copy_word_size)) uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < 0); if (pref_bias_copy_load - pref_bias_copy_store) { pg_addiu(&buf, A2, A0, pref_bias_copy_load - pref_bias_copy_store); uasm_l_copy_pref_store(&l, buf); off = 0; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < half_copy_loop_size); pg_addiu(&buf, A1, A1, 2 * off); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); if (off == -(4 * copy_word_size)) uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_store); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < 0); } if (pref_bias_copy_store) { pg_addiu(&buf, A2, A0, pref_bias_copy_store); uasm_l_copy_nopref(&l, buf); off = 0; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store(&buf, T0, off); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < half_copy_loop_size); pg_addiu(&buf, A1, A1, 2 * off); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store(&buf, T0, off); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); if (off == -(4 * copy_word_size)) uasm_il_bne(&buf, &r, A2, A0, label_copy_nopref); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < 0); } uasm_i_jr(&buf, RA); uasm_i_nop(&buf); BUG_ON(buf > copy_page_array + ARRAY_SIZE(copy_page_array)); uasm_resolve_relocs(relocs, labels); pr_debug("Synthesized copy page handler (%u instructions).\n", (u32)(buf - copy_page_array)); pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < (buf - copy_page_array); i++) pr_debug("\t.word 0x%08x\n", copy_page_array[i]); pr_debug("\t.set pop\n"); }
void __cpuinit build_clear_page(void) { int off; u32 *buf = (u32 *)&clear_page_array; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; int i; memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); if (current_cpu_data.cputype == CPU_CAVIUM_OCTEON2) { const unsigned int wb_nudge = 26; pg_addiu(&buf, T0, A0, PAGE_SIZE); UASM_i_ADDIU(&buf, A1, A0, 128); uasm_l_clear_pref(&l, buf); uasm_i_zcbt(&buf, A0); uasm_i_pref(&buf, wb_nudge, 0, A0); UASM_i_ADDIU(&buf, A0, A0, 256); uasm_i_zcbt(&buf, A1); uasm_i_pref(&buf, wb_nudge, 0, A1); UASM_i_ADDIU(&buf, A1, A1, 256); uasm_i_zcbt(&buf, A0); uasm_i_pref(&buf, wb_nudge, 0, A0); UASM_i_ADDIU(&buf, A0, A0, 256); uasm_i_zcbt(&buf, A1); uasm_i_pref(&buf, wb_nudge, 0, A1); UASM_i_ADDIU(&buf, A1, A1, 256); uasm_i_zcbt(&buf, A0); uasm_i_pref(&buf, wb_nudge, 0, A0); UASM_i_ADDIU(&buf, A0, A0, 256); uasm_i_zcbt(&buf, A1); uasm_i_pref(&buf, wb_nudge, 0, A1); UASM_i_ADDIU(&buf, A1, A1, 256); uasm_i_zcbt(&buf, A0); uasm_i_pref(&buf, wb_nudge, 0, A0); UASM_i_ADDIU(&buf, A0, A0, 256); uasm_i_zcbt(&buf, A1); uasm_i_pref(&buf, wb_nudge, 0, A1); uasm_il_bne(&buf, &r, A0, T0, label_clear_pref); UASM_i_ADDIU(&buf, A1, A1, 256); } else { set_prefetch_parameters(); /* * This algorithm makes the following assumptions: * - The prefetch bias is a multiple of 2 words. * - The prefetch bias is less than one page. */ BUG_ON(pref_bias_clear_store % (2 * clear_word_size)); BUG_ON(PAGE_SIZE < pref_bias_clear_store); off = PAGE_SIZE - pref_bias_clear_store; if (off > 0xffff || !pref_bias_clear_store) pg_addiu(&buf, A2, A0, off); else uasm_i_ori(&buf, A2, A0, off); if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) uasm_i_lui(&buf, AT, 0xa000); off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size) * cache_line_size : 0; while (off) { build_clear_pref(&buf, -off); off -= cache_line_size; } uasm_l_clear_pref(&l, buf); do { build_clear_pref(&buf, off); build_clear_store(&buf, off); off += clear_word_size; } while (off < half_clear_loop_size); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_clear_pref(&buf, off); if (off == -clear_word_size) uasm_il_bne(&buf, &r, A0, A2, label_clear_pref); build_clear_store(&buf, off); off += clear_word_size; } while (off < 0); if (pref_bias_clear_store) { pg_addiu(&buf, A2, A0, pref_bias_clear_store); uasm_l_clear_nopref(&l, buf); off = 0; do { build_clear_store(&buf, off); off += clear_word_size; } while (off < half_clear_loop_size); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { if (off == -clear_word_size) uasm_il_bne(&buf, &r, A0, A2, label_clear_nopref); build_clear_store(&buf, off); off += clear_word_size; } while (off < 0); } } uasm_i_jr(&buf, RA); uasm_i_nop(&buf); BUG_ON(buf > clear_page_array + ARRAY_SIZE(clear_page_array)); uasm_resolve_relocs(relocs, labels); pr_debug("Synthesized clear page handler (%u instructions).\n", (u32)(buf - clear_page_array)); pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < (buf - clear_page_array); i++) pr_debug("\t.word 0x%08x\n", clear_page_array[i]); pr_debug("\t.set pop\n"); }
void build_clear_page(void) { int off; u32 *buf = &__clear_page_start; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; int i; static atomic_t run_once = ATOMIC_INIT(0); if (atomic_xchg(&run_once, 1)) { return; } memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); set_prefetch_parameters(); /* * This algorithm makes the following assumptions: * - The prefetch bias is a multiple of 2 words. * - The prefetch bias is less than one page. */ BUG_ON(pref_bias_clear_store % (2 * clear_word_size)); BUG_ON(PAGE_SIZE < pref_bias_clear_store); off = PAGE_SIZE - pref_bias_clear_store; if (off > 0xffff || !pref_bias_clear_store) pg_addiu(&buf, A2, A0, off); else uasm_i_ori(&buf, A2, A0, off); if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000)); off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size) * cache_line_size : 0; while (off) { build_clear_pref(&buf, -off); off -= cache_line_size; } uasm_l_clear_pref(&l, buf); do { build_clear_pref(&buf, off); build_clear_store(&buf, off); off += clear_word_size; } while (off < half_clear_loop_size); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_clear_pref(&buf, off); if (off == -clear_word_size) uasm_il_bne(&buf, &r, A0, A2, label_clear_pref); build_clear_store(&buf, off); off += clear_word_size; } while (off < 0); if (pref_bias_clear_store) { pg_addiu(&buf, A2, A0, pref_bias_clear_store); uasm_l_clear_nopref(&l, buf); off = 0; do { build_clear_store(&buf, off); off += clear_word_size; } while (off < half_clear_loop_size); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { if (off == -clear_word_size) uasm_il_bne(&buf, &r, A0, A2, label_clear_nopref); build_clear_store(&buf, off); off += clear_word_size; } while (off < 0); } uasm_i_jr(&buf, RA); uasm_i_nop(&buf); BUG_ON(buf > &__clear_page_end); uasm_resolve_relocs(relocs, labels); pr_debug("Synthesized clear page handler (%u instructions).\n", (u32)(buf - &__clear_page_start)); pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < (buf - &__clear_page_start); i++) pr_debug("\t.word 0x%08x\n", (&__clear_page_start)[i]); pr_debug("\t.set pop\n"); }