void
omap4_pl310_init(platform_t plat, struct pl310_softc *sc)
{
uint32_t aux, prefetch;
aux = pl310_read4(sc, PL310_AUX_CTRL);
prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL);
/*
* Disable instruction prefetch
*/
prefetch &= ~PREFETCH_CTRL_INSTR_PREFETCH;
aux &= ~AUX_CTRL_INSTR_PREFETCH;
// prefetch &= ~PREFETCH_CTRL_DATA_PREFETCH;
// aux &= ~AUX_CTRL_DATA_PREFETCH;
/*
* Make sure data prefetch is on
*/
prefetch |= PREFETCH_CTRL_DATA_PREFETCH;
aux |= AUX_CTRL_DATA_PREFETCH;
/*
* TODO: add tunable for prefetch offset
* and experiment with performance
*/
ti_smc0(aux, 0, WRITE_AUXCTRL_REG);
ti_smc0(prefetch, 0, WRITE_PREFETCH_CTRL_REG);
}
void
platform_pl310_init(struct pl310_softc *sc)
{
uint32_t aux;
aux = pl310_read4(sc, PL310_AUX_CTRL);
/*
* The Amlogic Linux platform code enables via AUX:
*
* Early BRESP
* Full Line of Zero (which must match processor setting)
* Data Prefetch
*
* and additionally on the m6 enables:
*
* Instruction Prefetch
*
* For the moment we only enable Data Prefetch ...
* further refinements can happen as things mature.
*/
/*
* Disable instruction prefetch.
*/
aux &= ~AUX_CTRL_INSTR_PREFETCH;
/*
* Enable data prefetch.
*/
aux |= AUX_CTRL_DATA_PREFETCH;
pl310_write4(sc, PL310_AUX_CTRL, aux);
}
static __inline void
pl310_wait_background_op(uint32_t off, uint32_t mask)
{
while (pl310_read4(pl310_softc, off) & mask)
continue;
}
void
pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
uint32_t read, uint32_t write, uint32_t setup)
{
uint32_t v;
KASSERT(which_reg == PL310_TAG_RAM_CTRL ||
which_reg == PL310_DATA_RAM_CTRL,
("bad pl310 ram latency register address"));
v = pl310_read4(sc, which_reg);
if (setup != 0) {
KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup));
v &= ~RAM_CTRL_SETUP_MASK;
v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT;
}
if (read != 0) {
KASSERT(read <= 8, ("bad pl310 read latency: %d", read));
v &= ~RAM_CTRL_READ_MASK;
v |= (read - 1) << RAM_CTRL_READ_SHIFT;
}
if (write != 0) {
KASSERT(write <= 8, ("bad pl310 write latency: %d", write));
v &= ~RAM_CTRL_WRITE_MASK;
v |= (write - 1) << RAM_CTRL_WRITE_SHIFT;
}
pl310_write4(sc, which_reg, v);
}
static void
pl310_print_config(struct pl310_softc *sc)
{
uint32_t aux, prefetch;
const char *dis = "disabled";
const char *ena = "enabled";
aux = pl310_read4(sc, PL310_AUX_CTRL);
prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL);
device_printf(sc->sc_dev, "Early BRESP response: %s\n",
(aux & AUX_CTRL_EARLY_BRESP) ? ena : dis);
device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
(aux & AUX_CTRL_INSTR_PREFETCH) ? ena : dis);
device_printf(sc->sc_dev, "Data prefetch: %s\n",
(aux & AUX_CTRL_DATA_PREFETCH) ? ena : dis);
device_printf(sc->sc_dev, "Non-secure interrupt control: %s\n",
(aux & AUX_CTRL_NS_INT_CTRL) ? ena : dis);
device_printf(sc->sc_dev, "Non-secure lockdown: %s\n",
(aux & AUX_CTRL_NS_LOCKDOWN) ? ena : dis);
device_printf(sc->sc_dev, "Share override: %s\n",
(aux & AUX_CTRL_SHARE_OVERRIDE) ? ena : dis);
device_printf(sc->sc_dev, "Double linefill: %s\n",
(prefetch & PREFETCH_CTRL_DL) ? ena : dis);
device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
(prefetch & PREFETCH_CTRL_INSTR_PREFETCH) ? ena : dis);
device_printf(sc->sc_dev, "Data prefetch: %s\n",
(prefetch & PREFETCH_CTRL_DATA_PREFETCH) ? ena : dis);
device_printf(sc->sc_dev, "Double linefill on WRAP request: %s\n",
(prefetch & PREFETCH_CTRL_DL_ON_WRAP) ? ena : dis);
device_printf(sc->sc_dev, "Prefetch drop: %s\n",
(prefetch & PREFETCH_CTRL_PREFETCH_DROP) ? ena : dis);
device_printf(sc->sc_dev, "Incr double Linefill: %s\n",
(prefetch & PREFETCH_CTRL_INCR_DL) ? ena : dis);
device_printf(sc->sc_dev, "Not same ID on exclusive sequence: %s\n",
(prefetch & PREFETCH_CTRL_NOTSAMEID) ? ena : dis);
device_printf(sc->sc_dev, "Prefetch offset: %d\n",
(prefetch & PREFETCH_CTRL_OFFSET_MASK));
}
static void
pl310_set_way_sizes(struct pl310_softc *sc)
{
uint32_t aux_value;
aux_value = pl310_read4(sc, PL310_AUX_CTRL);
g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
AUX_CTRL_WAY_SIZE_SHIFT;
g_way_size = 1 << (g_way_size + 13);
if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
g_ways_assoc = 16;
else
g_ways_assoc = 8;
g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
g_l2cache_size = g_way_size * g_ways_assoc;
}
static int
pl310_filter(void *arg)
{
struct pl310_softc *sc = arg;
uint32_t intr;
intr = pl310_read4(sc, PL310_INTR_MASK);
if (!sc->sc_enabled && (intr & INTR_MASK_ECNTR)) {
/*
* This is for debug purpose, so be blunt about it
* We disable PL310 only when something fishy is going
* on and we need to make sure L2 cache is 100% disabled
*/
panic("pl310: caches disabled but cache event detected\n");
}
return (FILTER_HANDLED);
}
void
platform_pl310_init(struct pl310_softc *sc)
{
uint32_t reg;
/*
* Enable power saving modes:
* - Dynamic Gating stops the clock when the controller is idle.
*/
reg = pl310_read4(sc, PL310_POWER_CTRL);
reg |= POWER_CTRL_ENABLE_GATING;
pl310_write4(sc, PL310_POWER_CTRL, reg);
pl310_write4(sc, PL310_PREFETCH_CTRL, PREFETCH_CTRL_DL |
PREFETCH_CTRL_DATA_PREFETCH | PREFETCH_CTRL_INCR_DL |
PREFETCH_CTRL_DL_ON_WRAP);
/* Disable L2 cache sync for IO coherent operation */
sc->sc_io_coherent = true;
}
static int
pl310_attach(device_t dev)
{
struct pl310_softc *sc = device_get_softc(dev);
int rid = 0;
uint32_t aux_value;
uint32_t ctrl_value;
uint32_t cache_id;
sc->sc_dev = dev;
sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->sc_mem_res == NULL)
panic("%s: Cannot map registers", device_get_name(dev));
/* Allocate an IRQ resource */
rid = 0;
sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_ACTIVE | RF_SHAREABLE);
if (sc->sc_irq_res == NULL) {
panic("Cannot allocate IRQ\n");
}
pl310_softc = sc;
mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
sc->sc_enabled = pl310_enabled;
/* activate the interrupt */
bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
pl310_filter, NULL, sc, &sc->sc_irq_h);
cache_id = pl310_read4(sc, PL310_CACHE_ID);
sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) &
CACHE_ID_RELEASE_MASK;
device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
(cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
(cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
aux_value = pl310_read4(sc, PL310_AUX_CTRL);
g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
AUX_CTRL_WAY_SIZE_SHIFT;
g_way_size = 1 << (g_way_size + 13);
if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
g_ways_assoc = 16;
else
g_ways_assoc = 8;
g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
g_l2cache_size = g_way_size * g_ways_assoc;
/* Print the information */
device_printf(dev, "L2 Cache: %uKB/%dB %d ways\n", (g_l2cache_size / 1024),
g_l2cache_line_size, g_ways_assoc);
ctrl_value = pl310_read4(sc, PL310_CTRL);
if (sc->sc_enabled && !(ctrl_value & CTRL_ENABLED)) {
/* invalidate current content */
pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
pl310_wait_background_op(PL310_INV_WAY, 0xffff);
/* Enable the L2 cache if disabled */
platform_pl310_write_ctrl(sc, CTRL_ENABLED);
device_printf(dev, "L2 Cache enabled\n");
}
if (!sc->sc_enabled && (ctrl_value & CTRL_ENABLED)) {
/*
* Set counters so when cache event happens
* we'll get interrupt and be warned that something
* is off
*/
/* Cache Line Eviction for Counter 0 */
pl310_write4(sc, PL310_EVENT_COUNTER0_CONF,
EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_CO);
/* Data Read Request for Counter 1 */
pl310_write4(sc, PL310_EVENT_COUNTER1_CONF,
EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ);
/* Temporary switch on for final flush*/
sc->sc_enabled = 1;
pl310_wbinv_all();
sc->sc_enabled = 0;
platform_pl310_write_ctrl(sc, CTRL_DISABLED);
/* Enable and clear pending interrupts */
pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR);
pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL);
/* Enable counters and reset C0 and C1 */
pl310_write4(sc, PL310_EVENT_COUNTER_CTRL,
EVENT_COUNTER_CTRL_ENABLED |
EVENT_COUNTER_CTRL_C0_RESET |
EVENT_COUNTER_CTRL_C1_RESET);
device_printf(dev, "L2 Cache disabled\n");
}
if (sc->sc_enabled)
platform_pl310_init(sc);
pl310_wbinv_all();
/* Set the l2 functions in the set of cpufuncs */
cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
cpufuncs.cf_l2cache_inv_range = pl310_inv_range;
cpufuncs.cf_l2cache_wb_range = pl310_wb_range;
return (0);
}
static int
pl310_attach(device_t dev)
{
struct pl310_softc *sc = device_get_softc(dev);
int rid;
uint32_t cache_id, debug_ctrl;
phandle_t node;
sc->sc_dev = dev;
rid = 0;
sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->sc_mem_res == NULL)
panic("%s: Cannot map registers", device_get_name(dev));
/* Allocate an IRQ resource */
rid = 0;
sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_ACTIVE | RF_SHAREABLE);
if (sc->sc_irq_res == NULL) {
device_printf(dev, "cannot allocate IRQ, not using interrupt\n");
}
pl310_softc = sc;
mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
cache_id = pl310_read4(sc, PL310_CACHE_ID);
sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) &
CACHE_ID_RELEASE_MASK;
device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
(cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
(cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
/*
* Test for "arm,io-coherent" property and disable sync operation if
* platform is I/O coherent. Outer sync operations are not needed
* on coherent platform and may be harmful in certain situations.
*/
node = ofw_bus_get_node(dev);
if (OF_hasprop(node, "arm,io-coherent"))
sc->sc_io_coherent = true;
/*
* If L2 cache is already enabled then something has violated the rules,
* because caches are supposed to be off at kernel entry. The cache
* must be disabled to write the configuration registers without
* triggering an access error (SLVERR), but there's no documented safe
* procedure for disabling the L2 cache in the manual. So we'll try to
* invent one:
* - Use the debug register to force write-through mode and prevent
* linefills (allocation of new lines on read); now anything we do
* will not cause new data to come into the L2 cache.
* - Writeback and invalidate the current contents.
* - Disable the controller.
* - Restore the original debug settings.
*/
if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) {
device_printf(dev, "Warning: L2 Cache should not already be "
"active; trying to de-activate and re-initialize...\n");
sc->sc_enabled = 1;
debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL);
platform_pl310_write_debug(sc, debug_ctrl |
DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL);
pl310_set_way_sizes(sc);
pl310_wbinv_all();
platform_pl310_write_ctrl(sc, CTRL_DISABLED);
platform_pl310_write_debug(sc, debug_ctrl);
}
sc->sc_enabled = pl310_enabled;
if (sc->sc_enabled) {
platform_pl310_init(sc);
pl310_set_way_sizes(sc); /* platform init might change these */
pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
pl310_wait_background_op(PL310_INV_WAY, 0xffff);
platform_pl310_write_ctrl(sc, CTRL_ENABLED);
device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n",
(g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc);
if (bootverbose)
pl310_print_config(sc);
} else {
if (sc->sc_irq_res != NULL) {
sc->sc_ich = malloc(sizeof(*sc->sc_ich), M_DEVBUF, M_WAITOK);
sc->sc_ich->ich_func = pl310_config_intr;
sc->sc_ich->ich_arg = sc;
if (config_intrhook_establish(sc->sc_ich) != 0) {
device_printf(dev,
"config_intrhook_establish failed\n");
free(sc->sc_ich, M_DEVBUF);
return(ENXIO);
}
}
device_printf(dev, "L2 Cache disabled\n");
}
/* Set the l2 functions in the set of cpufuncs */
cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
cpufuncs.cf_l2cache_inv_range = pl310_inv_range;
cpufuncs.cf_l2cache_wb_range = pl310_wb_range;
cpufuncs.cf_l2cache_drain_writebuf = pl310_drain_writebuf;
return (0);
}