/* Activate a secondary processor. */
int __devinit start_secondary(void *unused)
{
unsigned int cpu = smp_processor_id();
struct device_node *l2_cache;
int i, base;
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
smp_store_cpu_info(cpu);
set_dec(tb_ticks_per_jiffy);
preempt_disable();
cpu_callin_map[cpu] = 1;
smp_ops->setup_cpu(cpu);
if (smp_ops->take_timebase)
smp_ops->take_timebase();
if (system_state > SYSTEM_BOOTING)
snapshot_timebase();
secondary_cpu_time_init();
ipi_call_lock();
notify_cpu_starting(cpu);
cpu_set(cpu, cpu_online_map);
/* Update sibling maps */
base = cpu_first_thread_in_core(cpu);
for (i = 0; i < threads_per_core; i++) {
if (cpu_is_offline(base + i))
continue;
cpu_set(cpu, per_cpu(cpu_sibling_map, base + i));
cpu_set(base + i, per_cpu(cpu_sibling_map, cpu));
/* cpu_core_map should be a superset of
* cpu_sibling_map even if we don't have cache
* information, so update the former here, too.
*/
cpu_set(cpu, per_cpu(cpu_core_map, base +i));
cpu_set(base + i, per_cpu(cpu_core_map, cpu));
}
l2_cache = cpu_to_l2cache(cpu);
for_each_online_cpu(i) {
struct device_node *np = cpu_to_l2cache(i);
if (!np)
continue;
if (np == l2_cache) {
cpu_set(cpu, per_cpu(cpu_core_map, i));
cpu_set(i, per_cpu(cpu_core_map, cpu));
}
of_node_put(np);
}
of_node_put(l2_cache);
ipi_call_unlock();
local_irq_enable();
cpu_idle();
return 0;
}
static unsigned int steal_context_smp(unsigned int id)
{
struct mm_struct *mm;
unsigned int cpu, max, i;
max = last_context - first_context;
/* Attempt to free next_context first and then loop until we manage */
while (max--) {
/* Pick up the victim mm */
mm = context_mm[id];
/* We have a candidate victim, check if it's active, on SMP
* we cannot steal active contexts
*/
if (mm->context.active) {
id++;
if (id > last_context)
id = first_context;
continue;
}
pr_hardcont(" | steal %d from 0x%p", id, mm);
/* Mark this mm has having no context anymore */
mm->context.id = MMU_NO_CONTEXT;
/* Mark it stale on all CPUs that used this mm. For threaded
* implementations, we set it on all threads on each core
* represented in the mask. A future implementation will use
* a core map instead but this will do for now.
*/
for_each_cpu(cpu, mm_cpumask(mm)) {
for (i = cpu_first_thread_in_core(cpu);
i <= cpu_last_thread_in_core(cpu); i++)
__set_bit(id, stale_map[i]);
cpu = i - 1;
}
return id;
}
/* This will happen if you have more CPUs than available contexts,
* all we can do here is wait a bit and try again
*/
raw_spin_unlock(&context_lock);
cpu_relax();
raw_spin_lock(&context_lock);
/* This will cause the caller to try again */
return MMU_NO_CONTEXT;
}
int __cpu_disable(void)
{
struct device_node *l2_cache;
int cpu = smp_processor_id();
int base, i;
int err;
if (!smp_ops->cpu_disable)
return -ENOSYS;
err = smp_ops->cpu_disable();
if (err)
return err;
/* Update sibling maps */
base = cpu_first_thread_in_core(cpu);
for (i = 0; i < threads_per_core; i++) {
cpumask_clear_cpu(cpu, cpu_sibling_mask(base + i));
cpumask_clear_cpu(base + i, cpu_sibling_mask(cpu));
cpumask_clear_cpu(cpu, cpu_core_mask(base + i));
cpumask_clear_cpu(base + i, cpu_core_mask(cpu));
}
l2_cache = cpu_to_l2cache(cpu);
for_each_present_cpu(i) {
struct device_node *np = cpu_to_l2cache(i);
if (!np)
continue;
if (np == l2_cache) {
cpumask_clear_cpu(cpu, cpu_core_mask(i));
cpumask_clear_cpu(i, cpu_core_mask(cpu));
}
of_node_put(np);
}
of_node_put(l2_cache);
return 0;
}
/* Activate a secondary processor. */
int __devinit start_secondary(void *unused)
{
unsigned int cpu = smp_processor_id();
struct device_node *l2_cache;
int i, base;
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
smp_store_cpu_info(cpu);
#if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
/* Clear any pending timer interrupts */
mtspr(SPRN_TSR, TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS);
/* Enable decrementer interrupt */
mtspr(SPRN_TCR, TCR_DIE);
#endif
set_dec(tb_ticks_per_jiffy);
preempt_disable();
cpu_callin_map[cpu] = 1;
if (smp_ops->setup_cpu)
smp_ops->setup_cpu(cpu);
if (smp_ops->take_timebase)
smp_ops->take_timebase();
if (system_state > SYSTEM_BOOTING)
snapshot_timebase();
secondary_cpu_time_init();
ipi_call_lock();
notify_cpu_starting(cpu);
set_cpu_online(cpu, true);
/* Update sibling maps */
base = cpu_first_thread_in_core(cpu);
for (i = 0; i < threads_per_core; i++) {
if (cpu_is_offline(base + i))
continue;
cpumask_set_cpu(cpu, cpu_sibling_mask(base + i));
cpumask_set_cpu(base + i, cpu_sibling_mask(cpu));
/* cpu_core_map should be a superset of
* cpu_sibling_map even if we don't have cache
* information, so update the former here, too.
*/
cpumask_set_cpu(cpu, cpu_core_mask(base + i));
cpumask_set_cpu(base + i, cpu_core_mask(cpu));
}
l2_cache = cpu_to_l2cache(cpu);
for_each_online_cpu(i) {
struct device_node *np = cpu_to_l2cache(i);
if (!np)
continue;
if (np == l2_cache) {
cpumask_set_cpu(cpu, cpu_core_mask(i));
cpumask_set_cpu(i, cpu_core_mask(cpu));
}
of_node_put(np);
}
of_node_put(l2_cache);
ipi_call_unlock();
local_irq_enable();
cpu_idle();
return 0;
}
void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
{
unsigned int i, id, cpu = smp_processor_id();
unsigned long *map;
/* No lockless fast path .. yet */
raw_spin_lock(&context_lock);
pr_hard("[%d] activating context for mm @%p, active=%d, id=%d",
cpu, next, next->context.active, next->context.id);
#ifdef CONFIG_SMP
/* Mark us active and the previous one not anymore */
next->context.active++;
if (prev) {
pr_hardcont(" (old=0x%p a=%d)", prev, prev->context.active);
WARN_ON(prev->context.active < 1);
prev->context.active--;
}
again:
#endif /* CONFIG_SMP */
/* If we already have a valid assigned context, skip all that */
id = next->context.id;
if (likely(id != MMU_NO_CONTEXT)) {
#ifdef DEBUG_MAP_CONSISTENCY
if (context_mm[id] != next)
pr_err("MMU: mm 0x%p has id %d but context_mm[%d] says 0x%p\n",
next, id, id, context_mm[id]);
#endif
goto ctxt_ok;
}
/* We really don't have a context, let's try to acquire one */
id = next_context;
if (id > last_context)
id = first_context;
map = context_map;
/* No more free contexts, let's try to steal one */
if (nr_free_contexts == 0) {
#ifdef CONFIG_SMP
if (num_online_cpus() > 1) {
id = steal_context_smp(id);
if (id == MMU_NO_CONTEXT)
goto again;
goto stolen;
}
#endif /* CONFIG_SMP */
id = steal_context_up(id);
goto stolen;
}
nr_free_contexts--;
/* We know there's at least one free context, try to find it */
while (__test_and_set_bit(id, map)) {
id = find_next_zero_bit(map, last_context+1, id);
if (id > last_context)
id = first_context;
}
stolen:
next_context = id + 1;
context_mm[id] = next;
next->context.id = id;
pr_hardcont(" | new id=%d,nrf=%d", id, nr_free_contexts);
context_check_map();
ctxt_ok:
/* If that context got marked stale on this CPU, then flush the
* local TLB for it and unmark it before we use it
*/
if (test_bit(id, stale_map[cpu])) {
pr_hardcont(" | stale flush %d [%d..%d]",
id, cpu_first_thread_in_core(cpu),
cpu_last_thread_in_core(cpu));
local_flush_tlb_mm(next);
/* XXX This clear should ultimately be part of local_flush_tlb_mm */
for (i = cpu_first_thread_in_core(cpu);
i <= cpu_last_thread_in_core(cpu); i++) {
__clear_bit(id, stale_map[i]);
}
}
/* Flick the MMU and release lock */
pr_hardcont(" -> %d\n", id);
set_context(id, next->pgd);
raw_spin_unlock(&context_lock);
}
