/** * menu_reflect - attempts to guess what happened after entry * @dev: the CPU * * NOTE: it's important to be fast here because this operation will add to * the overall exit latency. */ static void menu_reflect(struct cpuidle_device *dev) { struct menu_device *data = &__get_cpu_var(menu_devices); int last_idx = data->last_state_idx; unsigned int measured_us = cpuidle_get_last_residency(dev) + data->elapsed_us; struct cpuidle_state *target = &dev->states[last_idx]; /* * Ugh, this idle state doesn't support residency measurements, so we * are basically lost in the dark. As a compromise, assume we slept * for one full standard timer tick. However, be aware that this * could potentially result in a suboptimal state transition. */ if (!(target->flags & CPUIDLE_FLAG_TIME_VALID)) measured_us = USEC_PER_SEC / HZ; /* Predict time remaining until next break event */ if (measured_us + BREAK_FUZZ < data->expected_us - target->exit_latency) { data->predicted_us = max(measured_us, data->last_measured_us); data->last_measured_us = measured_us; data->elapsed_us = 0; } else { if (data->elapsed_us < data->elapsed_us + measured_us) data->elapsed_us = measured_us; else data->elapsed_us = -1; data->predicted_us = max(measured_us, data->last_measured_us); } }
/** * ladder_select_state - selects the next state to enter * @drv: cpuidle driver * @dev: the CPU */ static int ladder_select_state(struct cpuidle_driver *drv, struct cpuidle_device *dev) { struct ladder_device *ldev = this_cpu_ptr(&ladder_devices); struct ladder_device_state *last_state; int last_residency, last_idx = ldev->last_state_idx; int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY); /* Special case when user has set very strict latency requirement */ if (unlikely(latency_req == 0)) { ladder_do_selection(ldev, last_idx, 0); return 0; } last_state = &ldev->states[last_idx]; last_residency = cpuidle_get_last_residency(dev) - drv->states[last_idx].exit_latency; /* consider promotion */ if (last_idx < drv->state_count - 1 && !drv->states[last_idx + 1].disabled && !dev->states_usage[last_idx + 1].disable && last_residency > last_state->threshold.promotion_time && drv->states[last_idx + 1].exit_latency <= latency_req) { last_state->stats.promotion_count++; last_state->stats.demotion_count = 0; if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) { ladder_do_selection(ldev, last_idx, last_idx + 1); return last_idx + 1; } } /* consider demotion */ if (last_idx > CPUIDLE_DRIVER_STATE_START && (drv->states[last_idx].disabled || dev->states_usage[last_idx].disable || drv->states[last_idx].exit_latency > latency_req)) { int i; for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) { if (drv->states[i].exit_latency <= latency_req) break; } ladder_do_selection(ldev, last_idx, i); return i; } if (last_idx > CPUIDLE_DRIVER_STATE_START && last_residency < last_state->threshold.demotion_time) { last_state->stats.demotion_count++; last_state->stats.promotion_count = 0; if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) { ladder_do_selection(ldev, last_idx, last_idx - 1); return last_idx - 1; } } /* otherwise remain at the current state */ return last_idx; }
/** * ladder_select_state - selects the next state to enter * @dev: the CPU */ static int ladder_select_state(struct cpuidle_device *dev) { struct ladder_device *ldev = &__get_cpu_var(ladder_devices); struct ladder_device_state *last_state; int last_residency, last_idx = ldev->last_state_idx; int latency_req = pm_qos_requirement(PM_QOS_CPU_DMA_LATENCY); /* Special case when user has set very strict latency requirement */ if (unlikely(latency_req == 0)) { ladder_do_selection(ldev, last_idx, 0); return 0; } last_state = &ldev->states[last_idx]; if (dev->states[last_idx].flags & CPUIDLE_FLAG_TIME_VALID) last_residency = cpuidle_get_last_residency(dev) - dev->states[last_idx].exit_latency; else last_residency = last_state->threshold.promotion_time + 1; /* consider promotion */ if (last_idx < dev->state_count - 1 && last_residency > last_state->threshold.promotion_time && dev->states[last_idx + 1].exit_latency <= latency_req) { last_state->stats.promotion_count++; last_state->stats.demotion_count = 0; if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) { ladder_do_selection(ldev, last_idx, last_idx + 1); return last_idx + 1; } } /* consider demotion */ if (last_idx > CPUIDLE_DRIVER_STATE_START && dev->states[last_idx].exit_latency > latency_req) { int i; for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) { if (dev->states[i].exit_latency <= latency_req) break; } ladder_do_selection(ldev, last_idx, i); return i; } if (last_idx > CPUIDLE_DRIVER_STATE_START && last_residency < last_state->threshold.demotion_time) { last_state->stats.demotion_count++; last_state->stats.promotion_count = 0; if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) { ladder_do_selection(ldev, last_idx, last_idx - 1); return last_idx - 1; } } /* otherwise remain at the current state */ return last_idx; }
/** * menu_update - attempts to guess what happened after entry * @drv: cpuidle driver containing state data * @dev: the CPU */ static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) { struct menu_device *data = &__get_cpu_var(menu_devices); int last_idx = data->last_state_idx; unsigned int last_idle_us = cpuidle_get_last_residency(dev); struct cpuidle_state *target = &drv->states[last_idx]; unsigned int measured_us; unsigned int new_factor; /* * Ugh, this idle state doesn't support residency measurements, so we * are basically lost in the dark. As a compromise, assume we slept * for the whole expected time. */ if (unlikely(!(target->flags & CPUIDLE_FLAG_TIME_VALID))) last_idle_us = data->expected_us; measured_us = last_idle_us; /* * We correct for the exit latency; we are assuming here that the * exit latency happens after the event that we're interested in. */ if (measured_us > data->exit_us) measured_us -= data->exit_us; /* Update our correction ratio */ new_factor = data->correction_factor[data->bucket]; new_factor -= new_factor / DECAY; if (data->expected_us > 0 && measured_us < MAX_INTERESTING) new_factor += RESOLUTION * measured_us / data->expected_us; else /* * we were idle so long that we count it as a perfect * prediction */ new_factor += RESOLUTION; /* * We don't want 0 as factor; we always want at least * a tiny bit of estimated time. Fortunately, due to rounding, * new_factor will stay nonzero regardless of measured_us values * and the compiler can eliminate this test as long as DECAY > 1. */ if (DECAY == 1 && unlikely(new_factor == 0)) new_factor = 1; data->correction_factor[data->bucket] = new_factor; /* update the repeating-pattern data */ data->intervals[data->interval_ptr++] = last_idle_us; if (data->interval_ptr >= INTERVALS) data->interval_ptr = 0; }
/** * menu_reflect - records that data structures need update * @dev: the CPU * @index: the index of actual entered state * * NOTE: it's important to be fast here because this operation will add to * the overall exit latency. */ static void menu_reflect(struct cpuidle_device *dev, int index) { struct menu_device *data = &__get_cpu_var(menu_devices); data->last_state_idx = index; if (index >= 0) data->needs_update = 1; trace_printk("%d:: CPU%d: measured_us %u\n", ++global_counter[dev->cpu],dev->cpu, cpuidle_get_last_residency(dev)); }
static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) { struct menu_device *data = &__get_cpu_var(menu_devices); int last_idx = data->last_state_idx; unsigned int last_idle_us = cpuidle_get_last_residency(dev); struct cpuidle_state *target = &drv->states[last_idx]; unsigned int measured_us; u64 new_factor; if (unlikely(!(target->flags & CPUIDLE_FLAG_TIME_VALID))) last_idle_us = data->expected_us; measured_us = last_idle_us; if (measured_us > data->exit_us) measured_us -= data->exit_us; new_factor = data->correction_factor[data->bucket] * (DECAY - 1) / DECAY; if (data->expected_us > 0 && measured_us < MAX_INTERESTING) new_factor += RESOLUTION * measured_us / data->expected_us; else new_factor += RESOLUTION; if (new_factor == 0) new_factor = 1; data->correction_factor[data->bucket] = new_factor; data->intervals[data->interval_ptr++] = last_idle_us; if (data->interval_ptr >= INTERVALS) data->interval_ptr = 0; }
static int ladder_select_state(struct cpuidle_driver *drv, struct cpuidle_device *dev) { struct ladder_device *ldev = &__get_cpu_var(ladder_devices); struct ladder_device_state *last_state; int last_residency, last_idx = ldev->last_state_idx; int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY); if (unlikely(latency_req == 0)) { ladder_do_selection(ldev, last_idx, 0); return 0; } last_state = &ldev->states[last_idx]; if (drv->states[last_idx].flags & CPUIDLE_FLAG_TIME_VALID) { last_residency = cpuidle_get_last_residency(dev) - \ drv->states[last_idx].exit_latency; } else last_residency = last_state->threshold.promotion_time + 1; if (last_idx < drv->state_count - 1 && last_residency > last_state->threshold.promotion_time && drv->states[last_idx + 1].exit_latency <= latency_req) { last_state->stats.promotion_count++; last_state->stats.demotion_count = 0; if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) { ladder_do_selection(ldev, last_idx, last_idx + 1); return last_idx + 1; } } if (last_idx > CPUIDLE_DRIVER_STATE_START && drv->states[last_idx].exit_latency > latency_req) { int i; for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) { if (drv->states[i].exit_latency <= latency_req) break; } ladder_do_selection(ldev, last_idx, i); return i; } if (last_idx > CPUIDLE_DRIVER_STATE_START && last_residency < last_state->threshold.demotion_time) { last_state->stats.demotion_count++; last_state->stats.promotion_count = 0; if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) { ladder_do_selection(ldev, last_idx, last_idx - 1); return last_idx - 1; } } return last_idx; }
/** * menu_update - attempts to guess what happened after entry * @drv: cpuidle driver containing state data * @dev: the CPU */ static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) { struct menu_device *data = this_cpu_ptr(&menu_devices); int last_idx = data->last_state_idx; struct cpuidle_state *target = &drv->states[last_idx]; unsigned int measured_us; u64 new_factor; /* * Try to figure out how much time passed between entry to low * power state and occurrence of the wakeup event. * * If the entered idle state didn't support residency measurements, * we are basically lost in the dark how much time passed. * As a compromise, assume we slept for the whole expected time. * * Any measured amount of time will include the exit latency. * Since we are interested in when the wakeup begun, not when it * was completed, we must substract the exit latency. However, if * the measured amount of time is less than the exit latency, * assume the state was never reached and the exit latency is 0. */ if (unlikely(!(target->flags & CPUIDLE_FLAG_TIME_VALID))) { /* Use timer value as is */ measured_us = data->next_timer_us; } else { /* Use measured value */ measured_us = cpuidle_get_last_residency(dev); /* Deduct exit latency */ if (measured_us > target->exit_latency) measured_us -= target->exit_latency; /* Make sure our coefficients do not exceed unity */ if (measured_us > data->next_timer_us) measured_us = data->next_timer_us; } /* update our correction ratio */ new_factor = data->correction_factor[data->bucket] * (DECAY - 1) / DECAY; if (data->next_timer_us > 0 && measured_us < MAX_INTERESTING) new_factor += RESOLUTION * measured_us / data->next_timer_us; else /* * we were idle so long that we count it as a perfect * prediction */ new_factor += RESOLUTION; /* * We don't want 0 as factor; we always want at least * a tiny bit of estimated time. */ if (new_factor == 0) new_factor = 1; data->correction_factor[data->bucket] = new_factor; /* update the repeating-pattern data */ data->intervals[data->interval_ptr++] = measured_us; if (data->interval_ptr >= INTERVALS) data->interval_ptr = 0; }