/*
* __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
* This function should be called by the last man, after local CPU teardown
* is complete. CPU cache expected to be active.
*
* Returns:
* false: the critical section was not entered because an inbound CPU was
* observed, or the cluster is already being set up;
* true: the critical section was entered: it is now safe to tear down the
* cluster.
*/
bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
{
unsigned int i;
struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
/* Warn inbound CPUs that the cluster is being torn down: */
c->cluster = CLUSTER_GOING_DOWN;
sync_cache_w(&c->cluster);
/* Back out if the inbound cluster is already in the critical region: */
sync_cache_r(&c->inbound);
if (c->inbound == INBOUND_COMING_UP)
goto abort;
/*
* Wait for all CPUs to get out of the GOING_DOWN state, so that local
* teardown is complete on each CPU before tearing down the cluster.
*
* If any CPU has been woken up again from the DOWN state, then we
* shouldn't be taking the cluster down at all: abort in that case.
*/
sync_cache_r(&c->cpus);
for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
int cpustate;
if (i == cpu)
continue;
while (1) {
cpustate = c->cpus[i].cpu;
if (cpustate != CPU_GOING_DOWN)
break;
wfe();
sync_cache_r(&c->cpus[i].cpu);
}
switch (cpustate) {
case CPU_DOWN:
continue;
default:
goto abort;
}
}
return true;
abort:
__mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
return false;
}
static int __init nocache_trampoline(unsigned long _arg)
{
void (*cache_disable)(void) = (void *)_arg;
unsigned int mpidr = read_cpuid_mpidr();
unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
phys_reset_t phys_reset;
mcpm_set_entry_vector(cpu, cluster, cpu_resume);
setup_mm_for_reboot();
__mcpm_cpu_going_down(cpu, cluster);
BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
cache_disable();
__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
__mcpm_cpu_down(cpu, cluster);
phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
phys_reset(virt_to_phys(mcpm_entry_point));
BUG();
}
void mcpm_cpu_power_down(void)
{
unsigned int mpidr, cpu, cluster;
bool cpu_going_down, last_man;
phys_reset_t phys_reset;
mpidr = read_cpuid_mpidr();
cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
if (WARN_ON_ONCE(!platform_ops))
return;
BUG_ON(!irqs_disabled());
setup_mm_for_reboot();
__mcpm_cpu_going_down(cpu, cluster);
arch_spin_lock(&mcpm_lock);
BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
mcpm_cpu_use_count[cluster][cpu]--;
BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
mcpm_cpu_use_count[cluster][cpu] != 1);
cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
last_man = mcpm_cluster_unused(cluster);
if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
platform_ops->cpu_powerdown_prepare(cpu, cluster);
platform_ops->cluster_powerdown_prepare(cluster);
arch_spin_unlock(&mcpm_lock);
platform_ops->cluster_cache_disable();
__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
} else {
if (cpu_going_down)
platform_ops->cpu_powerdown_prepare(cpu, cluster);
arch_spin_unlock(&mcpm_lock);
/*
* If cpu_going_down is false here, that means a power_up
* request raced ahead of us. Even if we do not want to
* shut this CPU down, the caller still expects execution
* to return through the system resume entry path, like
* when the WFI is aborted due to a new IRQ or the like..
* So let's continue with cache cleaning in all cases.
*/
platform_ops->cpu_cache_disable();
}
__mcpm_cpu_down(cpu, cluster);
/* Now we are prepared for power-down, do it: */
if (cpu_going_down)
wfi();
/*
* It is possible for a power_up request to happen concurrently
* with a power_down request for the same CPU. In this case the
* CPU might not be able to actually enter a powered down state
* with the WFI instruction if the power_up request has removed
* the required reset condition. We must perform a re-entry in
* the kernel as if the power_up method just had deasserted reset
* on the CPU.
*/
phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
phys_reset(virt_to_phys(mcpm_entry_point));
/* should never get here */
BUG();
}
/*
* mmp_pm_down - Programs CPU to enter the specified state
*
* @addr: address points to the state selected by cpu governor
*
* Called from the CPUidle framework to program the device to the
* specified target state selected by the governor.
*/
static void mmp_pm_down(unsigned long addr)
{
int *idx = (int *)addr;
int mpidr, cpu, cluster;
bool skip_wfi = false, last_man = false;
mpidr = read_cpuid_mpidr();
cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
BUG_ON(cluster >= MAX_NR_CLUSTERS || cpu >= MAX_CPUS_PER_CLUSTER);
__mcpm_cpu_going_down(cpu, cluster);
arch_spin_lock(&mmp_lpm_lock);
mmp_pm_use_count[cluster][cpu]--;
if (mmp_pm_use_count[cluster][cpu] == 0) {
mmp_enter_lpm[cluster][cpu] = (1 << (*idx + 1)) - 1;
*idx = mmp_idle->cpudown_state;
if (cluster_is_idle(cluster)) {
cpu_cluster_pm_enter();
find_coupled_state(idx, cluster);
if (*idx >= mmp_idle->wakeup_state &&
*idx < mmp_idle->l2_flush_state &&
mmp_idle->ops->save_wakeup) {
mmp_wake_saved = 1;
mmp_idle->ops->save_wakeup();
}
BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
last_man = true;
}
if (mmp_idle->ops->set_pmu)
mmp_idle->ops->set_pmu(cpu, *idx);
} else if (mmp_pm_use_count[cluster][cpu] == 1) {
/*
* A power_up request went ahead of us.
* Even if we do not want to shut this CPU down,
* the caller expects a certain state as if the WFI
* was aborted. So let's continue with cache cleaning.
*/
skip_wfi = true;
*idx = INVALID_LPM;
} else
BUG();
if (last_man && (*idx >= mmp_idle->cpudown_state) && (*idx != LPM_D2_UDR)) {
cpu_dcstat_event(cpu_dcstat_clk, cpu, CPU_M2_OR_DEEPER_ENTER, *idx);
#ifdef CONFIG_VOLDC_STAT
vol_dcstat_event(VLSTAT_LPM_ENTRY, *idx, 0);
vol_ledstatus_event(*idx);
#endif
}
trace_pxa_cpu_idle(LPM_ENTRY(*idx), cpu, cluster);
cpu_dcstat_event(cpu_dcstat_clk, cpu, CPU_IDLE_ENTER, *idx);
if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
arch_spin_unlock(&mmp_lpm_lock);
__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
__mcpm_cpu_down(cpu, cluster);
if (*idx >= mmp_idle->l2_flush_state)
ca7_power_down_udr();
else
ca7_power_down();
} else {
arch_spin_unlock(&mmp_lpm_lock);
__mcpm_cpu_down(cpu, cluster);
ca7_power_down();
}
if (!skip_wfi)
cpu_do_idle();
}
