void UASM_i_LA_mostly(u32 **buf, unsigned int rs, long addr) { if (!uasm_in_compat_space_p(addr)) { uasm_i_lui(buf, rs, uasm_rel_highest(addr)); if (uasm_rel_higher(addr)) uasm_i_daddiu(buf, rs, rs, uasm_rel_higher(addr)); if (uasm_rel_hi(addr)) { uasm_i_dsll(buf, rs, rs, 16); uasm_i_daddiu(buf, rs, rs, uasm_rel_hi(addr)); uasm_i_dsll(buf, rs, rs, 16); } else uasm_i_dsll32(buf, rs, rs, 0); } else uasm_i_lui(buf, rs, uasm_rel_hi(addr)); }
void ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr) { if (!ISAFUNC(uasm_in_compat_space_p)(addr)) { ISAFUNC(uasm_i_lui)(buf, rs, uasm_rel_highest(addr)); if (uasm_rel_higher(addr)) ISAFUNC(uasm_i_daddiu)(buf, rs, rs, uasm_rel_higher(addr)); if (ISAFUNC(uasm_rel_hi(addr))) { ISAFUNC(uasm_i_dsll)(buf, rs, rs, 16); ISAFUNC(uasm_i_daddiu)(buf, rs, rs, ISAFUNC(uasm_rel_hi)(addr)); ISAFUNC(uasm_i_dsll)(buf, rs, rs, 16); } else ISAFUNC(uasm_i_dsll32)(buf, rs, rs, 0); } else ISAFUNC(uasm_i_lui)(buf, rs, ISAFUNC(uasm_rel_hi(addr))); }
static inline void pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) { if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) { if (off > 0x7fff) { uasm_i_lui(buf, T9, uasm_rel_hi(off)); uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off)); } else uasm_i_addiu(buf, T9, ZERO, off); uasm_i_daddu(buf, reg1, reg2, T9); } else { if (off > 0x7fff) { uasm_i_lui(buf, T9, uasm_rel_hi(off)); uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off)); UASM_i_ADDU(buf, reg1, reg2, T9); } else UASM_i_ADDIU(buf, reg1, reg2, off); } }
static void __init cps_gen_set_top_bit(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, unsigned r_addr, int lbl) { uasm_i_lui(pp, t0, uasm_rel_hi(0x80000000)); uasm_build_label(pl, *pp, lbl); uasm_i_ll(pp, t1, 0, r_addr); uasm_i_or(pp, t1, t1, t0); uasm_i_sc(pp, t1, 0, r_addr); uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); }
static void __cpuinit build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, unsigned int ptr) { long pgdc = (long)pgd_current; uasm_i_mfc0(p, pte, C0_BADVADDR); uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); uasm_i_srl(p, pte, pte, 22); /* load delay */ uasm_i_sll(p, pte, pte, 2); uasm_i_addu(p, ptr, ptr, pte); uasm_i_mfc0(p, pte, C0_CONTEXT); uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ uasm_i_addu(p, ptr, ptr, pte); uasm_i_lw(p, pte, 0, ptr); uasm_i_tlbp(p); /* load delay */ }
/* * The R3000 TLB handler is simple. */ static void __init build_r3000_tlb_refill_handler(void) { long pgdc = (long)pgd_current; u32 *p; int i; memset(tlb_handler, 0, sizeof(tlb_handler)); p = tlb_handler; uasm_i_mfc0(&p, K0, C0_BADVADDR); uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); uasm_i_srl(&p, K0, K0, 22); /* load delay */ uasm_i_sll(&p, K0, K0, 2); uasm_i_addu(&p, K1, K1, K0); uasm_i_mfc0(&p, K0, C0_CONTEXT); uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ uasm_i_addu(&p, K1, K1, K0); uasm_i_lw(&p, K0, 0, K1); uasm_i_nop(&p); /* load delay */ uasm_i_mtc0(&p, K0, C0_ENTRYLO0); uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ uasm_i_tlbwr(&p); /* cp0 delay */ uasm_i_jr(&p, K1); uasm_i_rfe(&p); /* branch delay */ if (p > tlb_handler + 32) panic("TLB refill handler space exceeded"); pr_debug("Wrote TLB refill handler (%u instructions).\n", (unsigned int)(p - tlb_handler)); pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < (p - tlb_handler); i++) pr_debug("\t.word 0x%08x\n", tlb_handler[i]); pr_debug("\t.set pop\n"); memcpy((void *)ebase, tlb_handler, 0x80); }
/* * BVADDR is the faulting address, PTR is scratch. * PTR will hold the pgd for vmalloc. */ static __init void build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int bvaddr, unsigned int ptr) { long swpd = (long)swapper_pg_dir; #ifdef MODULE_START long modd = (long)module_pg_dir; uasm_l_module_alloc(l, *p); /* * Assumption: * VMALLOC_START >= 0xc000000000000000UL * MODULE_START >= 0xe000000000000000UL */ UASM_i_SLL(p, ptr, bvaddr, 2); uasm_il_bgez(p, r, ptr, label_vmalloc); if (uasm_in_compat_space_p(MODULE_START) && !uasm_rel_lo(MODULE_START)) { uasm_i_lui(p, ptr, uasm_rel_hi(MODULE_START)); /* delay slot */ } else { /* unlikely configuration */ uasm_i_nop(p); /* delay slot */ UASM_i_LA(p, ptr, MODULE_START); } uasm_i_dsubu(p, bvaddr, bvaddr, ptr); if (uasm_in_compat_space_p(modd) && !uasm_rel_lo(modd)) { uasm_il_b(p, r, label_vmalloc_done); uasm_i_lui(p, ptr, uasm_rel_hi(modd)); } else { UASM_i_LA_mostly(p, ptr, modd); uasm_il_b(p, r, label_vmalloc_done); if (uasm_in_compat_space_p(modd)) uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(modd)); else uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(modd)); } uasm_l_vmalloc(l, *p); if (uasm_in_compat_space_p(MODULE_START) && !uasm_rel_lo(MODULE_START) && MODULE_START << 32 == VMALLOC_START) uasm_i_dsll32(p, ptr, ptr, 0); /* typical case */ else UASM_i_LA(p, ptr, VMALLOC_START); #else uasm_l_vmalloc(l, *p); UASM_i_LA(p, ptr, VMALLOC_START); #endif uasm_i_dsubu(p, bvaddr, bvaddr, ptr); if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { uasm_il_b(p, r, label_vmalloc_done); uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); } else { UASM_i_LA_mostly(p, ptr, swpd); uasm_il_b(p, r, label_vmalloc_done); if (uasm_in_compat_space_p(swpd)) uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); else uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); } }
#ifdef CONFIG_CPU_MICROMIPS if ((unsigned long)tlb_do_page_fault_1 & 1) { uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); uasm_i_jr(&p, K0); } else #endif uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
void build_copy_page(void) { int off; u32 *buf = &__copy_page_start; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; int i; static atomic_t run_once = ATOMIC_INIT(0); if (atomic_xchg(&run_once, 1)) { return; } memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); set_prefetch_parameters(); /* * This algorithm makes the following assumptions: * - All prefetch biases are multiples of 8 words. * - The prefetch biases are less than one page. * - The store prefetch bias isn't greater than the load * prefetch bias. */ BUG_ON(pref_bias_copy_load % (8 * copy_word_size)); BUG_ON(pref_bias_copy_store % (8 * copy_word_size)); BUG_ON(PAGE_SIZE < pref_bias_copy_load); BUG_ON(pref_bias_copy_store > pref_bias_copy_load); off = PAGE_SIZE - pref_bias_copy_load; if (off > 0xffff || !pref_bias_copy_load) pg_addiu(&buf, A2, A0, off); else uasm_i_ori(&buf, A2, A0, off); if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000)); off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) * cache_line_size : 0; while (off) { build_copy_load_pref(&buf, -off); off -= cache_line_size; } off = cache_line_size ? min(8, pref_bias_copy_store / cache_line_size) * cache_line_size : 0; while (off) { build_copy_store_pref(&buf, -off); off -= cache_line_size; } uasm_l_copy_pref_both(&l, buf); do { build_copy_load_pref(&buf, off); build_copy_load(&buf, T0, off); build_copy_load_pref(&buf, off + copy_word_size); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load_pref(&buf, off + 2 * copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load_pref(&buf, off + 3 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < half_copy_loop_size); pg_addiu(&buf, A1, A1, 2 * off); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_copy_load_pref(&buf, off); build_copy_load(&buf, T0, off); build_copy_load_pref(&buf, off + copy_word_size); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load_pref(&buf, off + 2 * copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load_pref(&buf, off + 3 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); if (off == -(4 * copy_word_size)) uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < 0); if (pref_bias_copy_load - pref_bias_copy_store) { pg_addiu(&buf, A2, A0, pref_bias_copy_load - pref_bias_copy_store); uasm_l_copy_pref_store(&l, buf); off = 0; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < half_copy_loop_size); pg_addiu(&buf, A1, A1, 2 * off); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store_pref(&buf, off); build_copy_store(&buf, T0, off); build_copy_store_pref(&buf, off + copy_word_size); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store_pref(&buf, off + 2 * copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store_pref(&buf, off + 3 * copy_word_size); if (off == -(4 * copy_word_size)) uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_store); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < 0); } if (pref_bias_copy_store) { pg_addiu(&buf, A2, A0, pref_bias_copy_store); uasm_l_copy_nopref(&l, buf); off = 0; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store(&buf, T0, off); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < half_copy_loop_size); pg_addiu(&buf, A1, A1, 2 * off); pg_addiu(&buf, A0, A0, 2 * off); off = -off; do { build_copy_load(&buf, T0, off); build_copy_load(&buf, T1, off + copy_word_size); build_copy_load(&buf, T2, off + 2 * copy_word_size); build_copy_load(&buf, T3, off + 3 * copy_word_size); build_copy_store(&buf, T0, off); build_copy_store(&buf, T1, off + copy_word_size); build_copy_store(&buf, T2, off + 2 * copy_word_size); if (off == -(4 * copy_word_size)) uasm_il_bne(&buf, &r, A2, A0, label_copy_nopref); build_copy_store(&buf, T3, off + 3 * copy_word_size); off += 4 * copy_word_size; } while (off < 0); } uasm_i_jr(&buf, RA); uasm_i_nop(&buf); BUG_ON(buf > &__copy_page_end); uasm_resolve_relocs(relocs, labels); pr_debug("Synthesized copy page handler (%u instructions).\n", (u32)(buf - &__copy_page_start)); pr_debug("\t.set push\n"); pr_debug("\t.set noreorder\n"); for (i = 0; i < (buf - &__copy_page_start); i++) pr_debug("\t.word 0x%08x\n", (&__copy_page_start)[i]); pr_debug("\t.set pop\n"); }