static void __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cache_desc *dcache, int lbl) { unsigned i, fsb_size = 8; unsigned num_loads = (fsb_size * 3) / 2; unsigned line_stride = 2; /* * Ensure that the fill/store buffer (FSB) is not holding the results * of a prefetch, since if it is then the CPC sequencer may become * stuck in the D3 (ClrBus) state whilst entering a low power state. */ /* TODO: this is interAptiv-specific, generalise it */ /* Preserve perf counter 1 setup */ uasm_i_mfc0(pp, t2, 25, 2); /* PerfCtl1 */ uasm_i_mfc0(pp, t3, 25, 3); /* PerfCnt1 */ /* Setup perf counter 1 to count FSB full pipeline stalls */ uasm_i_addiu(pp, t0, zero, 0x66f); uasm_i_mtc0(pp, t0, 25, 2); /* PerfCtl1 */ uasm_i_ehb(pp); uasm_i_mtc0(pp, zero, 25, 3); /* PerfCnt1 */ uasm_i_ehb(pp); /* Base address for loads */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Start of clear loop */ uasm_build_label(pl, *pp, lbl); /* Perform some loads to fill the FSB */ for (i = 0; i < num_loads; i++) uasm_i_lw(pp, zero, i * dcache->linesz * line_stride, t0); /* * Invalidate the new D-cache entries so that the cache will need * refilling (via the FSB) if the loop is executed again. */ for (i = 0; i < num_loads; i++) { uasm_i_cache(pp, Hit_Invalidate_D, i * dcache->linesz * line_stride, t0); uasm_i_cache(pp, Hit_Writeback_Inv_SD, i * dcache->linesz * line_stride, t0); } /* Completion barrier */ uasm_i_sync(pp, stype_memory); uasm_i_ehb(pp); /* Check whether the pipeline stalled due to the FSB being full */ uasm_i_mfc0(pp, t1, 25, 3); /* PerfCnt1 */ /* Loop if it didn't */ uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); /* Restore perf counter 1. The count may well now be wrong... */ uasm_i_mtc0(pp, t2, 25, 2); /* PerfCtl1 */ uasm_i_ehb(pp); uasm_i_mtc0(pp, t3, 25, 3); /* PerfCnt1 */ uasm_i_ehb(pp); }
static void * __init cps_gen_entry_code(unsigned cpu, enum cps_pm_state state) { struct uasm_label *l = labels; struct uasm_reloc *r = relocs; u32 *buf, *p; const unsigned r_online = a0; const unsigned r_nc_count = a1; const unsigned r_pcohctl = t7; const unsigned max_instrs = 256; unsigned cpc_cmd; enum { lbl_incready = 1, lbl_poll_cont, lbl_secondary_hang, lbl_disable_coherence, lbl_flush_fsb, lbl_invicache, lbl_flushdcache, lbl_hang, lbl_set_cont, lbl_secondary_cont, lbl_decready, }; /* Allocate a buffer to hold the generated code */ p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL); if (!buf) return NULL; /* Clear labels & relocs ready for (re)use */ memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); if (state == CPS_PM_POWER_GATED) { /* Power gating relies upon CPS SMP */ if (!mips_cps_smp_in_use()) goto out_err; /* * Save CPU state. Note the non-standard calling convention * with the return address placed in v0 to avoid clobbering * the ra register before it is saved. */ UASM_i_LA(&p, t0, (long)mips_cps_pm_save); uasm_i_jalr(&p, v0, t0); uasm_i_nop(&p); } /* * Load addresses of required CM & CPC registers. This is done early * because they're needed in both the enable & disable coherence steps * but in the coupled case the enable step will only run on one VPE. */ UASM_i_LA(&p, r_pcohctl, (long)_gcmp_base + GCMPCLCBOFS(COHCTL)); if (coupled_coherence) { /* Increment ready_count */ uasm_i_sync(&p, stype_ordering); uasm_build_label(&l, p, lbl_incready); uasm_i_ll(&p, t1, 0, r_nc_count); uasm_i_addiu(&p, t2, t1, 1); uasm_i_sc(&p, t2, 0, r_nc_count); uasm_il_beqz(&p, &r, t2, lbl_incready); uasm_i_addiu(&p, t1, t1, 1); /* Ordering barrier */ uasm_i_sync(&p, stype_ordering); /* * If this is the last VPE to become ready for non-coherence * then it should branch below. */ uasm_il_beq(&p, &r, t1, r_online, lbl_disable_coherence); uasm_i_nop(&p); if (state < CPS_PM_POWER_GATED) { /* * Otherwise this is not the last VPE to become ready * for non-coherence. It needs to wait until coherence * has been disabled before proceeding, which it will do * by polling for the top bit of ready_count being set. */ uasm_i_addiu(&p, t1, zero, -1); uasm_build_label(&l, p, lbl_poll_cont); uasm_i_lw(&p, t0, 0, r_nc_count); uasm_il_bltz(&p, &r, t0, lbl_secondary_cont); uasm_i_ehb(&p); uasm_i_yield(&p, zero, t1); uasm_il_b(&p, &r, lbl_poll_cont); uasm_i_nop(&p); } else { /* * The core will lose power & this VPE will not continue * so it can simply halt here. */ uasm_i_addiu(&p, t0, zero, TCHALT_H); uasm_i_mtc0(&p, t0, 2, 4); uasm_build_label(&l, p, lbl_secondary_hang); uasm_il_b(&p, &r, lbl_secondary_hang); uasm_i_nop(&p); } } /* * This is the point of no return - this VPE will now proceed to * disable coherence. At this point we *must* be sure that no other * VPE within the core will interfere with the L1 dcache. */ uasm_build_label(&l, p, lbl_disable_coherence); /* Invalidate the L1 icache */ cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache, Index_Invalidate_I, lbl_invicache); /* Writeback & invalidate the L1 dcache */ cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache, Index_Writeback_Inv_D, lbl_flushdcache); /* Completion barrier */ uasm_i_sync(&p, stype_memory); uasm_i_ehb(&p); /* * Disable all but self interventions. The load from COHCTL is defined * by the interAptiv & proAptiv SUMs as ensuring that the operation * resulting from the preceeding store is complete. */ uasm_i_addiu(&p, t0, zero, 1 << cpu_data[cpu].core); uasm_i_sw(&p, t0, 0, r_pcohctl); uasm_i_lw(&p, t0, 0, r_pcohctl); /* Sync to ensure previous interventions are complete */ uasm_i_sync(&p, stype_intervention); uasm_i_ehb(&p); /* Disable coherence */ uasm_i_sw(&p, zero, 0, r_pcohctl); uasm_i_lw(&p, t0, 0, r_pcohctl); if (state >= CPS_PM_CLOCK_GATED) { /* TODO: determine whether required based on CPC version */ cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu].dcache, lbl_flush_fsb); /* Determine the CPC command to issue */ switch (state) { case CPS_PM_CLOCK_GATED: cpc_cmd = CPC_Cx_CMD_CLOCKOFF; break; case CPS_PM_POWER_GATED: cpc_cmd = CPC_Cx_CMD_PWRDOWN; break; default: BUG(); goto out_err; } /* Issue the CPC command */ UASM_i_LA(&p, t0, (long)addr_cpc_cl_cmd()); uasm_i_addiu(&p, t1, zero, cpc_cmd); uasm_i_sw(&p, t1, 0, t0); if (state == CPS_PM_POWER_GATED) { /* If anything goes wrong just hang */ uasm_build_label(&l, p, lbl_hang); uasm_il_b(&p, &r, lbl_hang); uasm_i_nop(&p); /* * There's no point generating more code, the core is * powered down & if powered back up will run from the * reset vector not from here. */ goto gen_done; } /* Completion barrier */ uasm_i_sync(&p, stype_memory); uasm_i_ehb(&p); } if (state == CPS_PM_NC_WAIT) { /* * At this point it is safe for all VPEs to proceed with * execution. This VPE will set the top bit of ready_count * to indicate to the other VPEs that they may continue. */ if (coupled_coherence) cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); /* * VPEs which did not disable coherence will continue * executing, after coherence has been disabled, from this * point. */ uasm_build_label(&l, p, lbl_secondary_cont); /* Now perform our wait */ uasm_i_wait(&p, 0); } /* * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs * will run this. The first will actually re-enable coherence & the * rest will just be performing a rather unusual nop. */ uasm_i_addiu(&p, t0, zero, GCMP_CCB_COHCTL_DOMAIN_MSK); uasm_i_sw(&p, t0, 0, r_pcohctl); uasm_i_lw(&p, t0, 0, r_pcohctl); /* Completion barrier */ uasm_i_sync(&p, stype_memory); uasm_i_ehb(&p); if (coupled_coherence && (state == CPS_PM_NC_WAIT)) { /* Decrement ready_count */ uasm_build_label(&l, p, lbl_decready); uasm_i_sync(&p, stype_ordering); uasm_i_ll(&p, t1, 0, r_nc_count); uasm_i_addiu(&p, t2, t1, -1); uasm_i_sc(&p, t2, 0, r_nc_count); uasm_il_beqz(&p, &r, t2, lbl_decready); uasm_i_andi(&p, v0, t1, (1 << fls(smp_num_siblings)) - 1); /* Ordering barrier */ uasm_i_sync(&p, stype_ordering); } if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) { /* * At this point it is safe for all VPEs to proceed with * execution. This VPE will set the top bit of ready_count * to indicate to the other VPEs that they may continue. */ cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); /* * This core will be reliant upon another core sending a * power-up command to the CPC in order to resume operation. * Thus an arbitrary VPE can't trigger the core leaving the * idle state and the one that disables coherence might as well * be the one to re-enable it. The rest will continue from here * after that has been done. */ uasm_build_label(&l, p, lbl_secondary_cont); /* Ordering barrier */ uasm_i_sync(&p, stype_ordering); } /* The core is coherent, time to return to C code */ uasm_i_jr(&p, ra); uasm_i_nop(&p); gen_done: /* Ensure the code didn't exceed the resources allocated for it */ BUG_ON((p - buf) > max_instrs); BUG_ON((l - labels) > ARRAY_SIZE(labels)); BUG_ON((r - relocs) > ARRAY_SIZE(relocs)); /* Patch branch offsets */ uasm_resolve_relocs(relocs, labels); /* Flush the icache */ local_flush_icache_range((unsigned long)buf, (unsigned long)p); return buf; out_err: kfree(buf); return NULL; }
static int __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, struct uasm_reloc **pr, const struct cpuinfo_mips *cpu_info, int lbl) { unsigned i, fsb_size = 8; unsigned num_loads = (fsb_size * 3) / 2; unsigned line_stride = 2; unsigned line_size = cpu_info->dcache.linesz; unsigned perf_counter, perf_event; unsigned revision = cpu_info->processor_id & PRID_REV_MASK; /* * Determine whether this CPU requires an FSB flush, and if so which * performance counter/event reflect stalls due to a full FSB. */ switch (__get_cpu_type(cpu_info->cputype)) { case CPU_INTERAPTIV: perf_counter = 1; perf_event = 51; break; case CPU_PROAPTIV: /* Newer proAptiv cores don't require this workaround */ if (revision >= PRID_REV_ENCODE_332(1, 1, 0)) return 0; /* On older ones it's unavailable */ return -1; /* CPUs which do not require the workaround */ case CPU_P5600: case CPU_I6400: return 0; default: WARN_ONCE(1, "pm-cps: FSB flush unsupported for this CPU\n"); return -1; } /* * Ensure that the fill/store buffer (FSB) is not holding the results * of a prefetch, since if it is then the CPC sequencer may become * stuck in the D3 (ClrBus) state whilst entering a low power state. */ /* Preserve perf counter setup */ uasm_i_mfc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_mfc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ /* Setup perf counter to count FSB full pipeline stalls */ uasm_i_addiu(pp, t0, zero, (perf_event << 5) | 0xf); uasm_i_mtc0(pp, t0, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_ehb(pp); uasm_i_mtc0(pp, zero, 25, (perf_counter * 2) + 1); /* PerfCntN */ uasm_i_ehb(pp); /* Base address for loads */ UASM_i_LA(pp, t0, (long)CKSEG0); /* Start of clear loop */ uasm_build_label(pl, *pp, lbl); /* Perform some loads to fill the FSB */ for (i = 0; i < num_loads; i++) uasm_i_lw(pp, zero, i * line_size * line_stride, t0); /* * Invalidate the new D-cache entries so that the cache will need * refilling (via the FSB) if the loop is executed again. */ for (i = 0; i < num_loads; i++) { uasm_i_cache(pp, Hit_Invalidate_D, i * line_size * line_stride, t0); uasm_i_cache(pp, Hit_Writeback_Inv_SD, i * line_size * line_stride, t0); } /* Completion barrier */ uasm_i_sync(pp, stype_memory); uasm_i_ehb(pp); /* Check whether the pipeline stalled due to the FSB being full */ uasm_i_mfc0(pp, t1, 25, (perf_counter * 2) + 1); /* PerfCntN */ /* Loop if it didn't */ uasm_il_beqz(pp, pr, t1, lbl); uasm_i_nop(pp); /* Restore perf counter 1. The count may well now be wrong... */ uasm_i_mtc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ uasm_i_ehb(pp); uasm_i_mtc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ uasm_i_ehb(pp); return 0; }