/**
* kbase_destroy_context - Destroy a kernel base context.
* @kctx: Context to destroy
*
* Calls kbase_destroy_os_context() to free OS specific structures.
* Will release all outstanding regions.
*/
void kbase_destroy_context(struct kbase_context *kctx)
{
struct kbase_device *kbdev;
int pages;
unsigned long pending_regions_to_clean;
/* MALI_SEC_INTEGRATION */
int profile_count;
/* MALI_SEC_INTEGRATION */
if (!kctx) {
printk("An uninitialized or destroyed context is tried to be destroyed. kctx is null\n");
return ;
}
else if (kctx->ctx_status != CTX_INITIALIZED) {
printk("An uninitialized or destroyed context is tried to be destroyed\n");
printk("kctx: 0x%p, kctx->tgid: %d, kctx->ctx_status: 0x%x\n", kctx, kctx->tgid, kctx->ctx_status);
return ;
}
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
/* MALI_SEC_INTEGRATION */
for (profile_count = 0; profile_count < 3; profile_count++) {
if (wait_event_timeout(kctx->mem_profile_wait, atomic_read(&kctx->mem_profile_showing_state) == 0, (unsigned int) msecs_to_jiffies(1000)))
break;
else
printk("[G3D] waiting for memory profile\n");
}
/* MALI_SEC_INTEGRATION */
while (wait_event_timeout(kbdev->pm.suspending_wait, kbdev->pm.suspending == false, (unsigned int) msecs_to_jiffies(1000)) == 0)
printk("[G3D] Waiting for resuming the device\n");
KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u);
/* Ensure the core is powered up for the destroy process */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
kbase_jd_zap_context(kctx);
kbase_event_cleanup(kctx);
kbase_gpu_vm_lock(kctx);
/* MMU is disabled as part of scheduling out the context */
kbase_mmu_free_pgd(kctx);
/* drop the aliasing sink page now that it can't be mapped anymore */
kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false);
/* free pending region setups */
pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK;
while (pending_regions_to_clean) {
unsigned int cookie = __ffs(pending_regions_to_clean);
BUG_ON(!kctx->pending_regions[cookie]);
kbase_reg_pending_dtor(kctx->pending_regions[cookie]);
kctx->pending_regions[cookie] = NULL;
pending_regions_to_clean &= ~(1UL << cookie);
}
kbase_region_tracker_term(kctx);
kbase_gpu_vm_unlock(kctx);
/* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */
kbasep_js_kctx_term(kctx);
kbase_jd_exit(kctx);
kbase_pm_context_idle(kbdev);
kbase_mmu_term(kctx);
pages = atomic_read(&kctx->used_pages);
if (pages != 0)
dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
kbase_mem_pool_term(&kctx->mem_pool);
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
/* MALI_SEC_INTEGRATION */
if(kbdev->vendor_callbacks->destroy_context)
kbdev->vendor_callbacks->destroy_context(kctx);
if (kctx->ctx_need_qos) {
kctx->ctx_need_qos = false;
}
vfree(kctx);
/* MALI_SEC_INTEGRATION */
kctx = NULL;
}
void kbase_pm_change_policy(kbase_device *kbdev)
{
OSK_ASSERT(kbdev != NULL);
KBASE_TRACE_ADD( kbdev, PM_CHANGE_POLICY, NULL, NULL, 0u,
kbdev->pm.current_policy->id | (kbdev->pm.new_policy->id<<16) );
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->term(kbdev);
kbdev->pm.current_policy = kbdev->pm.new_policy;
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->init(kbdev);
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_POLICY_INIT);
/* Changing policy might have occurred during an update to the core desired
* states, but the KBASE_PM_EVENT_CHANGE_GPU_STATE event could've been
* optimized out if the previous policy had
* KBASE_PM_POLICY_FLAG_NO_CORE_TRANSITIONS set.
*
* In any case, we issue a KBASE_PM_EVENT_CHANGE_GPU_STATE just in case. */
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_CHANGE_GPU_STATE);
/* Now the policy change is finished, we release our fake context active reference */
kbase_pm_context_idle(kbdev);
kbdev->pm.new_policy = NULL;
}
/**
* @brief Clear the HW counters
*/
mali_error kbase_instr_hwcnt_clear(kbase_context *kctx)
{
unsigned long flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
kbase_device *kbdev;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
/* Check it's the context previously set up and we're not already dumping */
if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE)
goto out;
/* Clear the counters */
KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR, kctx);
err = MALI_ERROR_NONE;
out:
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
void kbase_backend_ctx_count_changed(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
unsigned long flags;
lockdep_assert_held(&js_devdata->runpool_mutex);
if (!timer_callback_should_run(kbdev)) {
/* Take spinlock to force synchronisation with timer */
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
backend->timer_running = false;
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
/* From now on, return value of timer_callback_should_run() will
* also cause the timer to not requeue itself. Its return value
* cannot change, because it depends on variables updated with
* the runpool_mutex held, which the caller of this must also
* hold */
hrtimer_cancel(&backend->scheduling_timer);
}
if (timer_callback_should_run(kbdev) && !backend->timer_running) {
/* Take spinlock to force synchronisation with timer */
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
backend->timer_running = true;
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
hrtimer_start(&backend->scheduling_timer,
HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
HRTIMER_MODE_REL);
KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u,
0u);
}
}
/**
* @brief Check whether a ctx has a certain attribute, and if so, release that
* attribute on the runpool.
*
* Requires:
* - jsctx mutex
* - runpool_irq spinlock
* - ctx is scheduled on the runpool
*
* @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
* In this state, the scheduler might be able to submit more jobs than
* previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
* or similar is called sometime later.
* @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
*/
STATIC mali_bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
mali_bool runpool_state_changed = MALI_FALSE;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
js_devdata = &kbdev->js_data;
js_kctx_info = &kctx->jctx.sched_info;
BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != MALI_FALSE);
if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != MALI_FALSE) {
KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0);
--(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]);
if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) {
/* Last de-refcount indicates a state change */
runpool_state_changed = MALI_TRUE;
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute);
}
}
return runpool_state_changed;
}
/**
* @brief Release a certain attribute on a ctx, also releasign it from the runpool
* if the context is scheduled.
*
* Requires:
* - jsctx mutex
* - If the context is scheduled, then runpool_irq spinlock must also be held
*
* @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
* This may allow the scheduler to submit more jobs than previously.
* @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
*/
STATIC mali_bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_kctx_info *js_kctx_info;
mali_bool runpool_state_changed = MALI_FALSE;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
js_kctx_info = &kctx->jctx.sched_info;
BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0);
if (js_kctx_info->ctx.is_scheduled != MALI_FALSE && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
/* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */
runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute);
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute);
}
/* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */
--(js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
return runpool_state_changed;
}
/**
* @brief Issue Cache Clean & Invalidate command to hardware
*/
static void kbasep_instr_hwcnt_cacheclean(kbase_device *kbdev)
{
unsigned long flags;
unsigned long pm_flags;
u32 irq_mask;
KBASE_DEBUG_ASSERT(NULL != kbdev);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
/* Wait for any reset to complete */
while (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.cache_clean_wait,
kbdev->hwcnt.state != KBASE_INSTR_STATE_RESETTING);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_REQUEST_CLEAN);
/* Enable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | CLEAN_CACHES_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES, NULL);
kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANING;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
}
void kbase_pm_request_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
{
unsigned long flags;
u64 cores;
kbase_pm_change_state change_gpu_state = 0u;
KBASE_DEBUG_ASSERT(kbdev != NULL);
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
cores = shader_cores;
while (cores) {
int bitnum = fls64(cores) - 1;
u64 bit = 1ULL << bitnum;
/* It should be almost impossible for this to overflow. It would require 2^32 atoms
* to request a particular core, which would require 2^24 contexts to submit. This
* would require an amount of memory that is impossible on a 32-bit system and
* extremely unlikely on a 64-bit system. */
int cnt = ++kbdev->shader_needed_cnt[bitnum];
if (1 == cnt) {
kbdev->shader_needed_bitmap |= bit;
change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
}
cores &= ~bit;
}
if (tiler_required != MALI_FALSE) {
++kbdev->tiler_needed_cnt;
KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
/* For tiler jobs, we must make sure that core 0 is not turned off if it's already on.
* However, it's safe for core 0 to be left off and turned on later whilst a tiler job
* is running. Hence, we don't need to update the cores state immediately. Also,
* attempts to turn off cores will always check the tiler_needed/inuse state first anyway.
*
* Finally, kbase_js_choose_affinity() ensures core 0 is always requested for tiler jobs
* anyway. Hence when there's only a tiler job in the system, this will still cause
* kbase_pm_update_cores_state_nolock() to be called.
*
* Note that we still need to keep track of tiler_needed/inuse_cnt, to ensure that
* kbase_pm_update_cores_state_nolock() can override the core availability policy and
* force core 0 to be powered when a tiler job is in the system. */
}
if (change_gpu_state) {
KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_START, change_gpu_state);
kbase_pm_update_cores_state_nolock(kbdev);
kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_END, change_gpu_state);
}
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
}
void kbase_pm_set_policy(kbase_device *kbdev, const kbase_pm_policy *new_policy)
{
OSK_ASSERT(kbdev != NULL);
OSK_ASSERT(new_policy != NULL);
if (kbdev->pm.new_policy) {
/* A policy change is already outstanding */
KBASE_TRACE_ADD( kbdev, PM_SET_POLICY, NULL, NULL, 0u, -1 );
return;
}
KBASE_TRACE_ADD( kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id );
/* During a policy change we pretend the GPU is active */
kbase_pm_context_active(kbdev);
kbdev->pm.new_policy = new_policy;
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_POLICY_CHANGE);
}
void kbase_pm_set_policy(kbase_device *kbdev, const kbase_pm_policy *new_policy)
{
const kbase_pm_policy *old_policy;
unsigned long flags;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(new_policy != NULL);
KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
/* During a policy change we pretend the GPU is active */
/* A suspend won't happen here, because we're in a syscall from a userspace thread */
kbase_pm_context_active(kbdev);
mutex_lock(&kbdev->pm.lock);
/* Remove the policy to prevent IRQ handlers from working on it */
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
old_policy = kbdev->pm.pm_current_policy;
kbdev->pm.pm_current_policy = NULL;
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, old_policy->id);
if (old_policy->term)
old_policy->term(kbdev);
KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, new_policy->id);
if (new_policy->init)
new_policy->init(kbdev);
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
kbdev->pm.pm_current_policy = new_policy;
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
/* If any core power state changes were previously attempted, but couldn't
* be made because the policy was changing (current_policy was NULL), then
* re-try them here. */
kbase_pm_update_active(kbdev);
kbase_pm_update_cores_state(kbdev);
mutex_unlock(&kbdev->pm.lock);
/* Now the policy change is finished, we release our fake context active reference */
kbase_pm_context_idle(kbdev);
}
void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
{
u64 desired_bitmap;
mali_bool cores_are_available;
lockdep_assert_held(&kbdev->pm.power_change_lock);
if (kbdev->pm.pm_current_policy == NULL)
return;
desired_bitmap = kbdev->pm.pm_current_policy->get_core_mask(kbdev);
desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
/* Enable core 0 if tiler required, regardless of core availability */
if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
desired_bitmap |= 1;
if (kbdev->pm.desired_shader_state != desired_bitmap)
KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, (u32)desired_bitmap);
/* Are any cores being powered on? */
if (~kbdev->pm.desired_shader_state & desired_bitmap ||
kbdev->pm.ca_in_transition != MALI_FALSE) {
/* Check if we are powering off any cores before updating shader state */
if (kbdev->pm.desired_shader_state & ~desired_bitmap) {
/* Start timer to power off cores */
kbdev->pm.shader_poweroff_pending |= (kbdev->pm.desired_shader_state & ~desired_bitmap);
kbdev->pm.shader_poweroff_pending_time = kbdev->pm.poweroff_shader_ticks;
}
kbdev->pm.desired_shader_state = desired_bitmap;
/* If any cores are being powered on, transition immediately */
cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
} else if (kbdev->pm.desired_shader_state & ~desired_bitmap) {
/* Start timer to power off cores */
kbdev->pm.shader_poweroff_pending |= (kbdev->pm.desired_shader_state & ~desired_bitmap);
kbdev->pm.shader_poweroff_pending_time = kbdev->pm.poweroff_shader_ticks;
} else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && kbdev->pm.poweroff_timer_running) {
/* If power policy is keeping cores on despite there being no active contexts
* then disable poweroff timer as it isn't required */
kbdev->pm.poweroff_timer_running = MALI_FALSE;
hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
}
/* Ensure timer does not power off wanted cores and make sure to power off unwanted cores */
if (kbdev->pm.shader_poweroff_pending != 0) {
kbdev->pm.shader_poweroff_pending &= ~(kbdev->pm.desired_shader_state & desired_bitmap);
if (kbdev->pm.shader_poweroff_pending == 0)
kbdev->pm.shader_poweroff_pending_time = 0;
}
/* Don't need 'cores_are_available', because we don't return anything */
CSTD_UNUSED(cores_are_available);
}
void kbase_pm_unrequest_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
{
unsigned long flags;
kbase_pm_change_state change_gpu_state = 0u;
KBASE_DEBUG_ASSERT(kbdev != NULL);
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
while (shader_cores) {
int bitnum = fls64(shader_cores) - 1;
u64 bit = 1ULL << bitnum;
int cnt;
KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
cnt = --kbdev->shader_needed_cnt[bitnum];
if (0 == cnt) {
kbdev->shader_needed_bitmap &= ~bit;
change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
}
shader_cores &= ~bit;
}
if (tiler_required != MALI_FALSE) {
KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
--kbdev->tiler_needed_cnt;
/* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
* extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
* when the last tiler job unrequests cores: kbase_js_choose_affinity() ensures core 0
* was originally requested for tiler jobs. Hence when there's only a tiler job in the
* system, this will still cause kbase_pm_update_cores_state_nolock() to be called. */
}
if (change_gpu_state) {
KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
kbase_pm_update_cores_state_nolock(kbdev);
/* Trace that any state change effectively completes immediately -
* no-one will wait on the state change */
kbase_pm_trace_check_and_finish_state_change(kbdev);
}
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
}
void kbase_pm_release_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
{
unsigned long flags;
kbase_pm_change_state change_gpu_state = 0u;
KBASE_DEBUG_ASSERT(kbdev != NULL);
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
while (shader_cores) {
int bitnum = fls64(shader_cores) - 1;
u64 bit = 1ULL << bitnum;
int cnt;
KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
cnt = --kbdev->shader_inuse_cnt[bitnum];
if (0 == cnt) {
kbdev->shader_inuse_bitmap &= ~bit;
change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
}
shader_cores &= ~bit;
}
if (tiler_required != MALI_FALSE) {
KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
--kbdev->tiler_inuse_cnt;
/* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
* extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
* when the last tiler job finishes: kbase_js_choose_affinity() ensures core 0 was
* originally requested for tiler jobs. Hence when there's only a tiler job in the
* system, this will still cause kbase_pm_update_cores_state_nolock() to be called */
}
if (change_gpu_state) {
KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_START, change_gpu_state);
kbase_pm_update_cores_state_nolock(kbdev);
kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_END, change_gpu_state);
/* Trace that any state change completed immediately */
kbase_pm_trace_check_and_finish_state_change(kbdev);
}
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
}
static void pm_callback_power_off(struct kbase_device *kbdev)
{
#if HARD_RESET_AT_POWER_OFF
/* Cause a GPU hard reset to test whether we have actually idled the GPU
* and that we properly reconfigure the GPU on power up.
* Usually this would be dangerous, but if the GPU is working correctly it should
* be completely safe as the GPU should not be active at this point.
* However this is disabled normally because it will most likely interfere with
* bus logging etc.
*/
KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
#endif
}
/**
* @brief Issue Dump command to hardware
*
* Notes:
* - does not sleep
*/
mali_error kbase_instr_hwcnt_dump_irq(struct kbase_context *kctx)
{
unsigned long flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
struct kbase_device *kbdev;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.kctx != kctx) {
/* The instrumentation has been setup for another context */
GPU_LOG(DVFS_INFO, DUMMY, 0u, 0u, "hwcnt irq error in %s %d \n", __FUNCTION__, err);
goto unlock;
}
if (kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE) {
/* HW counters are disabled or another dump is ongoing, or we're resetting */
GPU_LOG(DVFS_INFO, DUMMY, 0u, 0u, "hwcnt disabled or another dump is ongoing in %s %d \n", __FUNCTION__, err);
goto unlock;
}
kbdev->hwcnt.triggered = 0;
/* Mark that we're dumping - the PF handler can signal that we faulted */
kbdev->hwcnt.state = KBASE_INSTR_STATE_DUMPING;
/* Reconfigure the dump address */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kbdev->hwcnt.addr & 0xFFFFFFFF, NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kbdev->hwcnt.addr >> 32, NULL);
/* Start dumping */
KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, kbdev->hwcnt.addr, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE, kctx);
dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx);
err = MALI_ERROR_NONE;
unlock:
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
mali_error kbase_pm_powerup(kbase_device *kbdev)
{
unsigned long flags;
mali_error ret;
OSK_ASSERT(kbdev != NULL);
ret = kbase_pm_init_hw(kbdev);
if (ret != MALI_ERROR_NONE)
{
return ret;
}
kbase_pm_power_transitioning(kbdev);
kbasep_pm_read_present_cores(kbdev);
/* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
spin_lock_irqsave(&kbdev->pm.active_count_lock, flags);
kbdev->pm.active_count = 1;
spin_unlock_irqrestore(&kbdev->pm.active_count_lock, flags);
spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* Ensure cycle counter is off */
kbdev->pm.gpu_cycle_counter_requests = 0;
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
atomic_set(&kbdev->pm.pending_events, 0);
atomic_set(&kbdev->pm.work_active, KBASE_PM_WORK_ACTIVE_STATE_INACTIVE);
kbdev->pm.new_policy = NULL;
kbdev->pm.current_policy = policy_list[0];
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->init(kbdev);
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_POLICY_INIT);
/* Idle the GPU */
kbase_pm_context_idle(kbdev);
return MALI_ERROR_NONE;
}
void kbase_pm_term(kbase_device *kbdev)
{
unsigned long flags;
OSK_ASSERT(kbdev != NULL);
OSK_ASSERT(kbdev->pm.active_count == 0);
OSK_ASSERT(kbdev->pm.gpu_cycle_counter_requests == 0);
/* Destroy the workqueue - this ensures that all messages have been processed */
destroy_workqueue(kbdev->pm.workqueue);
if (kbdev->pm.current_policy != NULL)
{
/* Free any resources the policy allocated */
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->term(kbdev);
}
/* Synchronise with other threads */
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
/* Shut down the metrics subsystem */
kbasep_pm_metrics_term(kbdev);
}
STATIC void kbase_pm_worker(struct work_struct *data)
{
kbase_device *kbdev = container_of(data, kbase_device, pm.work);
int pending_events;
int old_value;
int i;
do
{
atomic_set(&kbdev->pm.work_active, KBASE_PM_WORK_ACTIVE_STATE_PROCESSING);
/* Atomically read and clear the bit mask */
pending_events = atomic_read(&kbdev->pm.pending_events);
do
{
old_value = pending_events;
pending_events = atomic_cmpxchg(&kbdev->pm.pending_events, old_value, 0);
} while (old_value != pending_events);
for(i = 0; pending_events; i++)
{
if (pending_events & (1 << i))
{
KBASE_TRACE_ADD( kbdev, PM_HANDLE_EVENT, NULL, NULL, 0u, i );
kbdev->pm.current_policy->event(kbdev, (kbase_pm_event)i);
pending_events &= ~(1 << i);
}
}
i = atomic_cmpxchg(&kbdev->pm.work_active,
KBASE_PM_WORK_ACTIVE_STATE_PROCESSING,
KBASE_PM_WORK_ACTIVE_STATE_INACTIVE);
} while (i == KBASE_PM_WORK_ACTIVE_STATE_PENDING_EVT);
kbdev->pm.no_outstanding_event = 1;
wake_up(&kbdev->pm.no_outstanding_event_wait);
}
static void pm_callback_power_off(struct kbase_device *kbdev)
{
struct device *dev = kbdev->dev;
int ret = 0, retry = 0;
#if HARD_RESET_AT_POWER_OFF
/* Cause a GPU hard reset to test whether we have actually idled the GPU
* and that we properly reconfigure the GPU on power up.
* Usually this would be dangerous, but if the GPU is working correctly it should
* be completely safe as the GPU should not be active at this point.
* However this is disabled normally because it will most likely interfere with
* bus logging etc.
*/
KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
#endif
if (unlikely(dev->power.disable_depth > 0)) {
kbase_platform_off(kbdev);
} else {
do {
ret = pm_schedule_suspend(dev, RUNTIME_PM_DELAY_TIME);
if (ret != -EAGAIN) {
if (unlikely(ret < 0)) {
pr_err("[mali-midgard] pm_schedule_suspend failed (%d)\n\n", ret);
WARN_ON(1);
}
/* correct status */
break;
}
/* -EAGAIN, repeated attempts for 1s totally */
msleep(50);
} while (++retry < 20);
}
}
/**
* @brief Retain a certain attribute on a ctx, also retaining it on the runpool
* if the context is scheduled.
*
* Requires:
* - jsctx mutex
* - If the context is scheduled, then runpool_irq spinlock must also be held
*
* @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
* This may allow the scheduler to submit more jobs than previously.
* @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
*/
STATIC mali_bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_kctx_info *js_kctx_info;
mali_bool runpool_state_changed = MALI_FALSE;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
js_kctx_info = &kctx->jctx.sched_info;
BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX);
++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
if (js_kctx_info->ctx.is_scheduled != MALI_FALSE && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
/* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute);
runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute);
}
return runpool_state_changed;
}
void kbase_pm_send_event(kbase_device *kbdev, kbase_pm_event event)
{
int pending_events;
int work_active;
int old_value, new_value;
OSK_ASSERT(kbdev != NULL);
if ( (kbdev->pm.current_policy->flags & KBASE_PM_POLICY_FLAG_NO_CORE_TRANSITIONS)
&& event == KBASE_PM_EVENT_CHANGE_GPU_STATE )
{
/* Optimize out event sending when the policy doesn't transition individual cores */
return;
}
KBASE_TRACE_ADD( kbdev, PM_SEND_EVENT, NULL, NULL, 0u, event );
pending_events = atomic_read(&kbdev->pm.pending_events);
/* Atomically OR the new event into the pending_events bit mask */
do
{
old_value = pending_events;
new_value = kbasep_pm_merge_event(pending_events, event);
if (old_value == new_value)
{
/* Event already pending */
return;
}
pending_events = atomic_cmpxchg(&kbdev->pm.pending_events, old_value, new_value);
} while (old_value != pending_events);
work_active = atomic_read(&kbdev->pm.work_active);
do
{
old_value = work_active;
switch(old_value)
{
case KBASE_PM_WORK_ACTIVE_STATE_INACTIVE:
/* Need to enqueue an event */
new_value = KBASE_PM_WORK_ACTIVE_STATE_ENQUEUED;
break;
case KBASE_PM_WORK_ACTIVE_STATE_ENQUEUED:
/* Event already queued */
return;
case KBASE_PM_WORK_ACTIVE_STATE_PROCESSING:
/* Event being processed, we need to ensure it checks for another event */
new_value = KBASE_PM_WORK_ACTIVE_STATE_PENDING_EVT;
break;
case KBASE_PM_WORK_ACTIVE_STATE_PENDING_EVT:
/* Event being processed, but another check for events is going to happen */
return;
default:
OSK_ASSERT(0);
}
work_active = atomic_cmpxchg(&kbdev->pm.work_active, old_value, new_value);
} while (old_value != work_active);
if (old_value == KBASE_PM_WORK_ACTIVE_STATE_INACTIVE)
{
KBASE_TRACE_ADD( kbdev, PM_ACTIVATE_WORKER, NULL, NULL, 0u, 0u );
kbdev->pm.no_outstanding_event = 0;
OSK_ASSERT(0 == object_is_on_stack(&kbdev->pm.work));
INIT_WORK(&kbdev->pm.work, kbase_pm_worker);
queue_work(kbdev->pm.workqueue, &kbdev->pm.work);
}
}
/**
* kbase_destroy_context - Destroy a kernel base context.
* @kctx: Context to destroy
*
* Calls kbase_destroy_os_context() to free OS specific structures.
* Will release all outstanding regions.
*/
void kbase_destroy_context(struct kbase_context *kctx)
{
struct kbase_device *kbdev;
int pages;
unsigned long pending_regions_to_clean;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u);
/* Ensure the core is powered up for the destroy process */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
kbase_jd_zap_context(kctx);
kbase_event_cleanup(kctx);
kbase_gpu_vm_lock(kctx);
/* MMU is disabled as part of scheduling out the context */
kbase_mmu_free_pgd(kctx);
/* drop the aliasing sink page now that it can't be mapped anymore */
kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false);
/* free pending region setups */
pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK;
while (pending_regions_to_clean) {
unsigned int cookie = __ffs(pending_regions_to_clean);
BUG_ON(!kctx->pending_regions[cookie]);
kbase_reg_pending_dtor(kctx->pending_regions[cookie]);
kctx->pending_regions[cookie] = NULL;
pending_regions_to_clean &= ~(1UL << cookie);
}
kbase_region_tracker_term(kctx);
kbase_gpu_vm_unlock(kctx);
/* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */
kbasep_js_kctx_term(kctx);
kbase_jd_exit(kctx);
kbase_pm_context_idle(kbdev);
kbase_mmu_term(kctx);
pages = atomic_read(&kctx->used_pages);
if (pages != 0)
dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
kbase_mem_pool_term(&kctx->mem_pool);
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
vfree(kctx);
}
kbase_pm_cores_ready kbase_pm_register_inuse_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
{
unsigned long flags;
u64 prev_shader_needed; /* Just for tracing */
u64 prev_shader_inuse; /* Just for tracing */
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
prev_shader_needed = kbdev->shader_needed_bitmap;
prev_shader_inuse = kbdev->shader_inuse_bitmap;
/* If desired_shader_state does not contain the requested cores, then power
* management is not attempting to powering those cores (most likely
* due to core availability policy) and a new job affinity must be
* chosen */
if ((kbdev->pm.desired_shader_state & shader_cores) != shader_cores) {
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
return KBASE_NEW_AFFINITY;
}
if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
(tiler_required != MALI_FALSE && !kbdev->tiler_available_bitmap)) {
/* Trace ongoing core transition */
kbase_timeline_pm_l2_transition_start(kbdev);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
return KBASE_CORES_NOT_READY;
}
/* If we started to trace a state change, then trace it has being finished
* by now, at the very latest */
kbase_pm_trace_check_and_finish_state_change(kbdev);
/* Trace core transition done */
kbase_timeline_pm_l2_transition_done(kbdev);
while (shader_cores) {
int bitnum = fls64(shader_cores) - 1;
u64 bit = 1ULL << bitnum;
int cnt;
KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
cnt = --kbdev->shader_needed_cnt[bitnum];
if (0 == cnt)
kbdev->shader_needed_bitmap &= ~bit;
/* shader_inuse_cnt should not overflow because there can only be a
* very limited number of jobs on the h/w at one time */
kbdev->shader_inuse_cnt[bitnum]++;
kbdev->shader_inuse_bitmap |= bit;
shader_cores &= ~bit;
}
if (tiler_required != MALI_FALSE) {
KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
--kbdev->tiler_needed_cnt;
kbdev->tiler_inuse_cnt++;
KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
}
if (prev_shader_needed != kbdev->shader_needed_bitmap)
KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
return KBASE_CORES_READY;
}
