static inline void build_copy_store_pref(u32 **buf, int off) { if (off & cache_line_mask()) return; if (pref_bias_copy_store) { uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off, A0); } else if (cache_line_size == (half_copy_loop_size << 1)) { if (cpu_has_cache_cdex_s) { uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); } else if (cpu_has_cache_cdex_p) { if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { uasm_i_nop(buf); uasm_i_nop(buf); uasm_i_nop(buf); uasm_i_nop(buf); } if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) uasm_i_lw(buf, ZERO, ZERO, AT); uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); } } }
static void __init cps_gen_cache_routine(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cache_desc *cache, unsigned op, int lbl) { unsigned cache_size = cache->ways << cache->waybit; unsigned i; const unsigned unroll_lines = 32; /* If the cache isn't present this function has it easy */ if (cache->flags & MIPS_CACHE_NOT_PRESENT) return; /* Load base address */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Calculate end address */ if (cache_size < 0x8000) uasm_i_addiu(pp, t1, t0, cache_size); else UASM_i_LA(pp, t1, (long)(CKSEG0 + cache_size)); /* Start of cache op loop */ uasm_build_label(pl, *pp, lbl); /* Generate the cache ops */ for (i = 0; i < unroll_lines; i++) { if (cpu_has_mips_r6) { uasm_i_cache(pp, op, 0, t0); uasm_i_addiu(pp, t0, t0, cache->linesz); } else { uasm_i_cache(pp, op, i * cache->linesz, t0); } } if (!cpu_has_mips_r6) /* Update the base address */ uasm_i_addiu(pp, t0, t0, unroll_lines * cache->linesz); /* Loop if we haven't reached the end address yet */ uasm_il_bne(pp, pr, t0, t1, lbl); uasm_i_nop(pp); }
static void __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cache_desc *dcache, int lbl) { unsigned i, fsb_size = 8; unsigned num_loads = (fsb_size * 3) / 2; unsigned line_stride = 2; /* * Ensure that the fill/store buffer (FSB) is not holding the results * of a prefetch, since if it is then the CPC sequencer may become * stuck in the D3 (ClrBus) state whilst entering a low power state. */ /* TODO: this is interAptiv-specific, generalise it */ /* Preserve perf counter 1 setup */ uasm_i_mfc0(pp, t2, 25, 2); /* PerfCtl1 */ uasm_i_mfc0(pp, t3, 25, 3); /* PerfCnt1 */ /* Setup perf counter 1 to count FSB full pipeline stalls */ uasm_i_addiu(pp, t0, zero, 0x66f); uasm_i_mtc0(pp, t0, 25, 2); /* PerfCtl1 */ uasm_i_ehb(pp); uasm_i_mtc0(pp, zero, 25, 3); /* PerfCnt1 */ uasm_i_ehb(pp); /* Base address for loads */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Start of clear loop */ uasm_build_label(pl, *pp, lbl); /* Perform some loads to fill the FSB */ for (i = 0; i < num_loads; i++) uasm_i_lw(pp, zero, i * dcache->linesz * line_stride, t0); /* * Invalidate the new D-cache entries so that the cache will need * refilling (via the FSB) if the loop is executed again. */ for (i = 0; i < num_loads; i++) { uasm_i_cache(pp, Hit_Invalidate_D, i * dcache->linesz * line_stride, t0); uasm_i_cache(pp, Hit_Writeback_Inv_SD, i * dcache->linesz * line_stride, t0); } /* Completion barrier */ uasm_i_sync(pp, stype_memory); uasm_i_ehb(pp); /* Check whether the pipeline stalled due to the FSB being full */ uasm_i_mfc0(pp, t1, 25, 3); /* PerfCnt1 */ /* Loop if it didn't */ uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); /* Restore perf counter 1. The count may well now be wrong... */ uasm_i_mtc0(pp, t2, 25, 2); /* PerfCtl1 */ uasm_i_ehb(pp); uasm_i_mtc0(pp, t3, 25, 3); /* PerfCnt1 */ uasm_i_ehb(pp); }
static int __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cpuinfo_mips *cpu_info, int lbl) { unsigned i, fsb_size = 8; unsigned num_loads = (fsb_size * 3) / 2; unsigned line_stride = 2; unsigned line_size = cpu_info->dcache.linesz; unsigned perf_counter, perf_event; unsigned revision = cpu_info->processor_id & PRID_REV_MASK; /* * Determine whether this CPU requires an FSB flush, and if so which * performance counter/event reflect stalls due to a full FSB. */ switch (__get_cpu_type(cpu_info->cputype)) { case CPU_INTERAPTIV: perf_counter = 1; perf_event = 51; break; case CPU_PROAPTIV: /* Newer proAptiv cores don't require this workaround */ if (revision >= PRID_REV_ENCODE_332(1, 1, 0)) return 0; /* On older ones it's unavailable */ return -1; /* CPUs which do not require the workaround */ case CPU_P5600: case CPU_I6400: return 0; default: WARN_ONCE(1, "pm-cps: FSB flush unsupported for this CPU\n"); return -1; } /* * Ensure that the fill/store buffer (FSB) is not holding the results * of a prefetch, since if it is then the CPC sequencer may become * stuck in the D3 (ClrBus) state whilst entering a low power state. */ /* Preserve perf counter setup */ uasm_i_mfc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_mfc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ /* Setup perf counter to count FSB full pipeline stalls */ uasm_i_addiu(pp, t0, zero, (perf_event << 5) | 0xf); uasm_i_mtc0(pp, t0, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_ehb(pp); uasm_i_mtc0(pp, zero, 25, (perf_counter * 2) + 1); /* PerfCntN */ uasm_i_ehb(pp); /* Base address for loads */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Start of clear loop */ uasm_build_label(pl, *pp, lbl); /* Perform some loads to fill the FSB */ for (i = 0; i < num_loads; i++) uasm_i_lw(pp, zero, i * line_size * line_stride, t0); /* * Invalidate the new D-cache entries so that the cache will need * refilling (via the FSB) if the loop is executed again. */ for (i = 0; i < num_loads; i++) { uasm_i_cache(pp, Hit_Invalidate_D, i * line_size * line_stride, t0); uasm_i_cache(pp, Hit_Writeback_Inv_SD, i * line_size * line_stride, t0); } /* Completion barrier */ uasm_i_sync(pp, stype_memory); uasm_i_ehb(pp); /* Check whether the pipeline stalled due to the FSB being full */ uasm_i_mfc0(pp, t1, 25, (perf_counter * 2) + 1); /* PerfCntN */ /* Loop if it didn't */ uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); /* Restore perf counter 1. The count may well now be wrong... */ uasm_i_mtc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_ehb(pp); uasm_i_mtc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ uasm_i_ehb(pp); return 0; }