/**
* menu_reflect - attempts to guess what happened after entry
* @dev: the CPU
*
* NOTE: it's important to be fast here because this operation will add to
* the overall exit latency.
*/
static void menu_reflect(struct cpuidle_device *dev)
{
struct menu_device *data = &__get_cpu_var(menu_devices);
int last_idx = data->last_state_idx;
unsigned int measured_us =
cpuidle_get_last_residency(dev) + data->elapsed_us;
struct cpuidle_state *target = &dev->states[last_idx];
/*
* Ugh, this idle state doesn't support residency measurements, so we
* are basically lost in the dark. As a compromise, assume we slept
* for one full standard timer tick. However, be aware that this
* could potentially result in a suboptimal state transition.
*/
if (!(target->flags & CPUIDLE_FLAG_TIME_VALID))
measured_us = USEC_PER_SEC / HZ;
/* Predict time remaining until next break event */
if (measured_us + BREAK_FUZZ < data->expected_us - target->exit_latency) {
data->predicted_us = max(measured_us, data->last_measured_us);
data->last_measured_us = measured_us;
data->elapsed_us = 0;
} else {
if (data->elapsed_us < data->elapsed_us + measured_us)
data->elapsed_us = measured_us;
else
data->elapsed_us = -1;
data->predicted_us = max(measured_us, data->last_measured_us);
}
}
/**
* ladder_select_state - selects the next state to enter
* @drv: cpuidle driver
* @dev: the CPU
*/
static int ladder_select_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev)
{
struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
struct ladder_device_state *last_state;
int last_residency, last_idx = ldev->last_state_idx;
int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
/* Special case when user has set very strict latency requirement */
if (unlikely(latency_req == 0)) {
ladder_do_selection(ldev, last_idx, 0);
return 0;
}
last_state = &ldev->states[last_idx];
last_residency = cpuidle_get_last_residency(dev) - drv->states[last_idx].exit_latency;
/* consider promotion */
if (last_idx < drv->state_count - 1 &&
!drv->states[last_idx + 1].disabled &&
!dev->states_usage[last_idx + 1].disable &&
last_residency > last_state->threshold.promotion_time &&
drv->states[last_idx + 1].exit_latency <= latency_req) {
last_state->stats.promotion_count++;
last_state->stats.demotion_count = 0;
if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
ladder_do_selection(ldev, last_idx, last_idx + 1);
return last_idx + 1;
}
}
/* consider demotion */
if (last_idx > CPUIDLE_DRIVER_STATE_START &&
(drv->states[last_idx].disabled ||
dev->states_usage[last_idx].disable ||
drv->states[last_idx].exit_latency > latency_req)) {
int i;
for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) {
if (drv->states[i].exit_latency <= latency_req)
break;
}
ladder_do_selection(ldev, last_idx, i);
return i;
}
if (last_idx > CPUIDLE_DRIVER_STATE_START &&
last_residency < last_state->threshold.demotion_time) {
last_state->stats.demotion_count++;
last_state->stats.promotion_count = 0;
if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
ladder_do_selection(ldev, last_idx, last_idx - 1);
return last_idx - 1;
}
}
/* otherwise remain at the current state */
return last_idx;
}
/**
* ladder_select_state - selects the next state to enter
* @dev: the CPU
*/
static int ladder_select_state(struct cpuidle_device *dev)
{
struct ladder_device *ldev = &__get_cpu_var(ladder_devices);
struct ladder_device_state *last_state;
int last_residency, last_idx = ldev->last_state_idx;
int latency_req = pm_qos_requirement(PM_QOS_CPU_DMA_LATENCY);
/* Special case when user has set very strict latency requirement */
if (unlikely(latency_req == 0)) {
ladder_do_selection(ldev, last_idx, 0);
return 0;
}
last_state = &ldev->states[last_idx];
if (dev->states[last_idx].flags & CPUIDLE_FLAG_TIME_VALID)
last_residency = cpuidle_get_last_residency(dev) - dev->states[last_idx].exit_latency;
else
last_residency = last_state->threshold.promotion_time + 1;
/* consider promotion */
if (last_idx < dev->state_count - 1 &&
last_residency > last_state->threshold.promotion_time &&
dev->states[last_idx + 1].exit_latency <= latency_req) {
last_state->stats.promotion_count++;
last_state->stats.demotion_count = 0;
if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
ladder_do_selection(ldev, last_idx, last_idx + 1);
return last_idx + 1;
}
}
/* consider demotion */
if (last_idx > CPUIDLE_DRIVER_STATE_START &&
dev->states[last_idx].exit_latency > latency_req) {
int i;
for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) {
if (dev->states[i].exit_latency <= latency_req)
break;
}
ladder_do_selection(ldev, last_idx, i);
return i;
}
if (last_idx > CPUIDLE_DRIVER_STATE_START &&
last_residency < last_state->threshold.demotion_time) {
last_state->stats.demotion_count++;
last_state->stats.promotion_count = 0;
if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
ladder_do_selection(ldev, last_idx, last_idx - 1);
return last_idx - 1;
}
}
/* otherwise remain at the current state */
return last_idx;
}
/**
* menu_update - attempts to guess what happened after entry
* @drv: cpuidle driver containing state data
* @dev: the CPU
*/
static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
{
struct menu_device *data = &__get_cpu_var(menu_devices);
int last_idx = data->last_state_idx;
unsigned int last_idle_us = cpuidle_get_last_residency(dev);
struct cpuidle_state *target = &drv->states[last_idx];
unsigned int measured_us;
unsigned int new_factor;
/*
* Ugh, this idle state doesn't support residency measurements, so we
* are basically lost in the dark. As a compromise, assume we slept
* for the whole expected time.
*/
if (unlikely(!(target->flags & CPUIDLE_FLAG_TIME_VALID)))
last_idle_us = data->expected_us;
measured_us = last_idle_us;
/*
* We correct for the exit latency; we are assuming here that the
* exit latency happens after the event that we're interested in.
*/
if (measured_us > data->exit_us)
measured_us -= data->exit_us;
/* Update our correction ratio */
new_factor = data->correction_factor[data->bucket];
new_factor -= new_factor / DECAY;
if (data->expected_us > 0 && measured_us < MAX_INTERESTING)
new_factor += RESOLUTION * measured_us / data->expected_us;
else
/*
* we were idle so long that we count it as a perfect
* prediction
*/
new_factor += RESOLUTION;
/*
* We don't want 0 as factor; we always want at least
* a tiny bit of estimated time. Fortunately, due to rounding,
* new_factor will stay nonzero regardless of measured_us values
* and the compiler can eliminate this test as long as DECAY > 1.
*/
if (DECAY == 1 && unlikely(new_factor == 0))
new_factor = 1;
data->correction_factor[data->bucket] = new_factor;
/* update the repeating-pattern data */
data->intervals[data->interval_ptr++] = last_idle_us;
if (data->interval_ptr >= INTERVALS)
data->interval_ptr = 0;
}
/**
* menu_reflect - records that data structures need update
* @dev: the CPU
* @index: the index of actual entered state
*
* NOTE: it's important to be fast here because this operation will add to
* the overall exit latency.
*/
static void menu_reflect(struct cpuidle_device *dev, int index)
{
struct menu_device *data = &__get_cpu_var(menu_devices);
data->last_state_idx = index;
if (index >= 0)
data->needs_update = 1;
trace_printk("%d:: CPU%d: measured_us %u\n",
++global_counter[dev->cpu],dev->cpu,
cpuidle_get_last_residency(dev));
}
static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
{
struct menu_device *data = &__get_cpu_var(menu_devices);
int last_idx = data->last_state_idx;
unsigned int last_idle_us = cpuidle_get_last_residency(dev);
struct cpuidle_state *target = &drv->states[last_idx];
unsigned int measured_us;
u64 new_factor;
if (unlikely(!(target->flags & CPUIDLE_FLAG_TIME_VALID)))
last_idle_us = data->expected_us;
measured_us = last_idle_us;
if (measured_us > data->exit_us)
measured_us -= data->exit_us;
new_factor = data->correction_factor[data->bucket]
* (DECAY - 1) / DECAY;
if (data->expected_us > 0 && measured_us < MAX_INTERESTING)
new_factor += RESOLUTION * measured_us / data->expected_us;
else
new_factor += RESOLUTION;
if (new_factor == 0)
new_factor = 1;
data->correction_factor[data->bucket] = new_factor;
data->intervals[data->interval_ptr++] = last_idle_us;
if (data->interval_ptr >= INTERVALS)
data->interval_ptr = 0;
}
static int ladder_select_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev)
{
struct ladder_device *ldev = &__get_cpu_var(ladder_devices);
struct ladder_device_state *last_state;
int last_residency, last_idx = ldev->last_state_idx;
int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
if (unlikely(latency_req == 0)) {
ladder_do_selection(ldev, last_idx, 0);
return 0;
}
last_state = &ldev->states[last_idx];
if (drv->states[last_idx].flags & CPUIDLE_FLAG_TIME_VALID) {
last_residency = cpuidle_get_last_residency(dev) - \
drv->states[last_idx].exit_latency;
}
else
last_residency = last_state->threshold.promotion_time + 1;
if (last_idx < drv->state_count - 1 &&
last_residency > last_state->threshold.promotion_time &&
drv->states[last_idx + 1].exit_latency <= latency_req) {
last_state->stats.promotion_count++;
last_state->stats.demotion_count = 0;
if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
ladder_do_selection(ldev, last_idx, last_idx + 1);
return last_idx + 1;
}
}
if (last_idx > CPUIDLE_DRIVER_STATE_START &&
drv->states[last_idx].exit_latency > latency_req) {
int i;
for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) {
if (drv->states[i].exit_latency <= latency_req)
break;
}
ladder_do_selection(ldev, last_idx, i);
return i;
}
if (last_idx > CPUIDLE_DRIVER_STATE_START &&
last_residency < last_state->threshold.demotion_time) {
last_state->stats.demotion_count++;
last_state->stats.promotion_count = 0;
if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
ladder_do_selection(ldev, last_idx, last_idx - 1);
return last_idx - 1;
}
}
return last_idx;
}
/**
* menu_update - attempts to guess what happened after entry
* @drv: cpuidle driver containing state data
* @dev: the CPU
*/
static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
{
struct menu_device *data = this_cpu_ptr(&menu_devices);
int last_idx = data->last_state_idx;
struct cpuidle_state *target = &drv->states[last_idx];
unsigned int measured_us;
u64 new_factor;
/*
* Try to figure out how much time passed between entry to low
* power state and occurrence of the wakeup event.
*
* If the entered idle state didn't support residency measurements,
* we are basically lost in the dark how much time passed.
* As a compromise, assume we slept for the whole expected time.
*
* Any measured amount of time will include the exit latency.
* Since we are interested in when the wakeup begun, not when it
* was completed, we must substract the exit latency. However, if
* the measured amount of time is less than the exit latency,
* assume the state was never reached and the exit latency is 0.
*/
if (unlikely(!(target->flags & CPUIDLE_FLAG_TIME_VALID))) {
/* Use timer value as is */
measured_us = data->next_timer_us;
} else {
/* Use measured value */
measured_us = cpuidle_get_last_residency(dev);
/* Deduct exit latency */
if (measured_us > target->exit_latency)
measured_us -= target->exit_latency;
/* Make sure our coefficients do not exceed unity */
if (measured_us > data->next_timer_us)
measured_us = data->next_timer_us;
}
/* update our correction ratio */
new_factor = data->correction_factor[data->bucket]
* (DECAY - 1) / DECAY;
if (data->next_timer_us > 0 && measured_us < MAX_INTERESTING)
new_factor += RESOLUTION * measured_us / data->next_timer_us;
else
/*
* we were idle so long that we count it as a perfect
* prediction
*/
new_factor += RESOLUTION;
/*
* We don't want 0 as factor; we always want at least
* a tiny bit of estimated time.
*/
if (new_factor == 0)
new_factor = 1;
data->correction_factor[data->bucket] = new_factor;
/* update the repeating-pattern data */
data->intervals[data->interval_ptr++] = measured_us;
if (data->interval_ptr >= INTERVALS)
data->interval_ptr = 0;
}
