/**
* kbase_destroy_context - Destroy a kernel base context.
* @kctx: Context to destroy
*
* Calls kbase_destroy_os_context() to free OS specific structures.
* Will release all outstanding regions.
*/
void kbase_destroy_context(struct kbase_context *kctx)
{
struct kbase_device *kbdev;
int pages;
unsigned long pending_regions_to_clean;
/* MALI_SEC_INTEGRATION */
int profile_count;
/* MALI_SEC_INTEGRATION */
if (!kctx) {
printk("An uninitialized or destroyed context is tried to be destroyed. kctx is null\n");
return ;
}
else if (kctx->ctx_status != CTX_INITIALIZED) {
printk("An uninitialized or destroyed context is tried to be destroyed\n");
printk("kctx: 0x%p, kctx->tgid: %d, kctx->ctx_status: 0x%x\n", kctx, kctx->tgid, kctx->ctx_status);
return ;
}
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
/* MALI_SEC_INTEGRATION */
for (profile_count = 0; profile_count < 3; profile_count++) {
if (wait_event_timeout(kctx->mem_profile_wait, atomic_read(&kctx->mem_profile_showing_state) == 0, (unsigned int) msecs_to_jiffies(1000)))
break;
else
printk("[G3D] waiting for memory profile\n");
}
/* MALI_SEC_INTEGRATION */
while (wait_event_timeout(kbdev->pm.suspending_wait, kbdev->pm.suspending == false, (unsigned int) msecs_to_jiffies(1000)) == 0)
printk("[G3D] Waiting for resuming the device\n");
KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u);
/* Ensure the core is powered up for the destroy process */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
kbase_jd_zap_context(kctx);
kbase_event_cleanup(kctx);
kbase_gpu_vm_lock(kctx);
/* MMU is disabled as part of scheduling out the context */
kbase_mmu_free_pgd(kctx);
/* drop the aliasing sink page now that it can't be mapped anymore */
kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false);
/* free pending region setups */
pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK;
while (pending_regions_to_clean) {
unsigned int cookie = __ffs(pending_regions_to_clean);
BUG_ON(!kctx->pending_regions[cookie]);
kbase_reg_pending_dtor(kctx->pending_regions[cookie]);
kctx->pending_regions[cookie] = NULL;
pending_regions_to_clean &= ~(1UL << cookie);
}
kbase_region_tracker_term(kctx);
kbase_gpu_vm_unlock(kctx);
/* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */
kbasep_js_kctx_term(kctx);
kbase_jd_exit(kctx);
kbase_pm_context_idle(kbdev);
kbase_mmu_term(kctx);
pages = atomic_read(&kctx->used_pages);
if (pages != 0)
dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
kbase_mem_pool_term(&kctx->mem_pool);
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
/* MALI_SEC_INTEGRATION */
if(kbdev->vendor_callbacks->destroy_context)
kbdev->vendor_callbacks->destroy_context(kctx);
if (kctx->ctx_need_qos) {
kctx->ctx_need_qos = false;
}
vfree(kctx);
/* MALI_SEC_INTEGRATION */
kctx = NULL;
}
int kbase_instr_hwcnt_enable(struct kbase_context *kctx,
struct kbase_uk_hwcnt_setup *setup)
{
struct kbase_device *kbdev;
bool access_allowed;
int err;
kbdev = kctx->kbdev;
/* Determine if the calling task has access to this capability */
access_allowed = kbase_security_has_capability(kctx,
KBASE_SEC_INSTR_HW_COUNTERS_COLLECT,
KBASE_SEC_FLAG_NOAUDIT);
if (!access_allowed)
return -EINVAL;
/* Mark the context as active so the GPU is kept turned on */
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread. */
kbase_pm_context_active(kbdev);
/* Schedule the context in */
kbasep_js_schedule_privileged_ctx(kbdev, kctx);
err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, setup);
if (err) {
/* Release the context. This had its own Power Manager Active
* reference */
kbasep_js_release_privileged_ctx(kbdev, kctx);
/* Also release our Power Manager Active reference */
kbase_pm_context_idle(kbdev);
}
return err;
}
void kbase_wait_write_flush(struct kbase_context *kctx)
{
u32 base_count = 0;
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread */
kbase_pm_context_active(kctx->kbdev);
kbase_pm_request_gpu_cycle_counter(kctx->kbdev);
while (true) {
u32 new_count;
new_count = kbase_reg_read(kctx->kbdev,
GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
/* First time around, just store the count. */
if (base_count == 0) {
base_count = new_count;
continue;
}
/* No need to handle wrapping, unsigned maths works for this. */
if ((new_count - base_count) > 1000)
break;
}
kbase_pm_release_gpu_cycle_counter(kctx->kbdev);
kbase_pm_context_idle(kctx->kbdev);
}
void kbase_pm_resume(struct kbase_device *kbdev)
{
int nr_keep_gpu_powered_ctxs;
/* MUST happen before any pm_context_active calls occur */
mutex_lock(&kbdev->pm.lock);
kbdev->pm.suspending = MALI_FALSE;
mutex_unlock(&kbdev->pm.lock);
/* Initial active call, to power on the GPU/cores if needed */
kbase_pm_context_active(kbdev);
/* Restore the keep_gpu_powered calls */
for (nr_keep_gpu_powered_ctxs = atomic_read(&kbdev->keep_gpu_powered_count);
nr_keep_gpu_powered_ctxs > 0 ;
--nr_keep_gpu_powered_ctxs ) {
kbase_pm_context_active(kbdev);
}
/* Re-enable instrumentation, if it was previously disabled */
kbase_instr_hwcnt_resume(kbdev);
/* Resume any blocked atoms (which may cause contexts to be scheduled in
* and dependent atoms to run) */
kbase_resume_suspended_soft_jobs(kbdev);
/* Resume the Job Scheduler and associated components, and start running
* atoms */
kbasep_js_resume(kbdev);
/* Matching idle call, to power off the GPU/cores if we didn't actually
* need it and the policy doesn't want it on */
kbase_pm_context_idle(kbdev);
}
void kbase_pm_change_policy(kbase_device *kbdev)
{
OSK_ASSERT(kbdev != NULL);
KBASE_TRACE_ADD( kbdev, PM_CHANGE_POLICY, NULL, NULL, 0u,
kbdev->pm.current_policy->id | (kbdev->pm.new_policy->id<<16) );
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->term(kbdev);
kbdev->pm.current_policy = kbdev->pm.new_policy;
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->init(kbdev);
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_POLICY_INIT);
/* Changing policy might have occurred during an update to the core desired
* states, but the KBASE_PM_EVENT_CHANGE_GPU_STATE event could've been
* optimized out if the previous policy had
* KBASE_PM_POLICY_FLAG_NO_CORE_TRANSITIONS set.
*
* In any case, we issue a KBASE_PM_EVENT_CHANGE_GPU_STATE just in case. */
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_CHANGE_GPU_STATE);
/* Now the policy change is finished, we release our fake context active reference */
kbase_pm_context_idle(kbdev);
kbdev->pm.new_policy = NULL;
}
int kbase_instr_hwcnt_enable(struct kbase_context *kctx,
struct kbase_uk_hwcnt_setup *setup)
{
struct kbase_device *kbdev;
int err;
kbdev = kctx->kbdev;
/* Mark the context as active so the GPU is kept turned on */
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread. */
kbase_pm_context_active(kbdev);
/* Schedule the context in */
kbasep_js_schedule_privileged_ctx(kbdev, kctx);
err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, setup);
if (err) {
/* Release the context. This had its own Power Manager Active
* reference */
kbasep_js_release_privileged_ctx(kbdev, kctx);
/* Also release our Power Manager Active reference */
kbase_pm_context_idle(kbdev);
}
return err;
}
mali_error kbase_pm_powerup(struct kbase_device *kbdev)
{
unsigned long flags;
mali_error ret;
KBASE_DEBUG_ASSERT(kbdev != NULL);
mutex_lock(&kbdev->pm.lock);
/* A suspend won't happen during startup/insmod */
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
/* MALI_SEC_INTEGRATION */
/* while the GPU initialization, vendor desired gpu log will be out by set_power_dbg(FALSE) calls */
if(kbdev->vendor_callbacks->set_poweron_dbg)
kbdev->vendor_callbacks->set_poweron_dbg(FALSE);
/* Power up the GPU, don't enable IRQs as we are not ready to receive them. */
ret = kbase_pm_init_hw(kbdev, MALI_FALSE);
if (ret != MALI_ERROR_NONE) {
mutex_unlock(&kbdev->pm.lock);
return ret;
}
kbasep_pm_read_present_cores(kbdev);
kbdev->pm.debug_core_mask = kbdev->shader_present_bitmap;
/* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
kbdev->pm.active_count = 1;
spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* Ensure cycle counter is off */
kbdev->pm.gpu_cycle_counter_requests = 0;
spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* We are ready to receive IRQ's now as power policy is set up, so enable them now. */
#ifdef CONFIG_MALI_DEBUG
spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
kbdev->pm.driver_ready_for_irqs = MALI_TRUE;
spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
#endif
kbase_pm_enable_interrupts(kbdev);
/* Turn on the GPU and any cores needed by the policy */
kbase_pm_do_poweron(kbdev, MALI_FALSE);
mutex_unlock(&kbdev->pm.lock);
/* MALI_SEC_INTEGRATION */
if(kbdev->vendor_callbacks->hwcnt_init)
kbdev->vendor_callbacks->hwcnt_init(kbdev);
/* Idle the GPU and/or cores, if the policy wants it to */
kbase_pm_context_idle(kbdev);
return MALI_ERROR_NONE;
}
void kbase_pm_suspend(struct kbase_device *kbdev)
{
int nr_keep_gpu_powered_ctxs;
KBASE_DEBUG_ASSERT(kbdev);
/* MALI_SEC_INTEGRATION */
if(kbdev->vendor_callbacks->hwcnt_prepare_suspend)
kbdev->vendor_callbacks->hwcnt_prepare_suspend(kbdev);
mutex_lock(&kbdev->pm.lock);
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
kbdev->pm.suspending = MALI_TRUE;
mutex_unlock(&kbdev->pm.lock);
/* From now on, the active count will drop towards zero. Sometimes, it'll
* go up briefly before going down again. However, once it reaches zero it
* will stay there - guaranteeing that we've idled all pm references */
/* Suspend job scheduler and associated components, so that it releases all
* the PM active count references */
kbasep_js_suspend(kbdev);
#ifndef MALI_SEC_HWCNT
/* Suspend any counter collection that might be happening */
kbase_instr_hwcnt_suspend(kbdev);
#endif
/* Cancel the keep_gpu_powered calls */
for (nr_keep_gpu_powered_ctxs = atomic_read(&kbdev->keep_gpu_powered_count);
nr_keep_gpu_powered_ctxs > 0;
--nr_keep_gpu_powered_ctxs) {
kbase_pm_context_idle(kbdev);
}
/* Wait for the active count to reach zero. This is not the same as
* waiting for a power down, since not all policies power down when this
* reaches zero. */
wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0);
/* NOTE: We synchronize with anything that was just finishing a
* kbase_pm_context_idle() call by locking the pm.lock below */
/* Force power off the GPU and all cores (regardless of policy), only after
* the PM active count reaches zero (otherwise, we risk turning it off
* prematurely) */
mutex_lock(&kbdev->pm.lock);
kbase_pm_cancel_deferred_poweroff(kbdev);
kbase_pm_do_poweroff(kbdev, MALI_TRUE);
mutex_unlock(&kbdev->pm.lock);
}
void kbase_pm_ca_set_policy(struct kbase_device *kbdev,
const struct kbase_pm_ca_policy *new_policy)
{
const struct kbase_pm_ca_policy *old_policy;
unsigned long flags;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(new_policy != NULL);
KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u,
new_policy->id);
/* During a policy change we pretend the GPU is active */
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread */
kbase_pm_context_active(kbdev);
mutex_lock(&kbdev->pm.lock);
/* Remove the policy to prevent IRQ handlers from working on it */
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
old_policy = kbdev->pm.backend.ca_current_policy;
kbdev->pm.backend.ca_current_policy = NULL;
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
if (old_policy->term)
old_policy->term(kbdev);
if (new_policy->init)
new_policy->init(kbdev);
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
kbdev->pm.backend.ca_current_policy = new_policy;
/* If any core power state changes were previously attempted, but
* couldn't be made because the policy was changing (current_policy was
* NULL), then re-try them here. */
kbase_pm_update_cores_state_nolock(kbdev);
kbdev->pm.backend.ca_current_policy->update_core_status(kbdev,
kbdev->shader_ready_bitmap,
kbdev->shader_transitioning_bitmap);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
mutex_unlock(&kbdev->pm.lock);
/* Now the policy change is finished, we release our fake context active
* reference */
kbase_pm_context_idle(kbdev);
}
void kbase_pm_resume(struct kbase_device *kbdev)
{
int nr_keep_gpu_powered_ctxs;
/* MUST happen before any pm_context_active calls occur */
mutex_lock(&kbdev->pm.lock);
kbdev->pm.suspending = MALI_FALSE;
kbase_pm_do_poweron(kbdev, MALI_TRUE);
mutex_unlock(&kbdev->pm.lock);
/* Initial active call, to power on the GPU/cores if needed */
kbase_pm_context_active(kbdev);
/* Restore the keep_gpu_powered calls */
for (nr_keep_gpu_powered_ctxs = atomic_read(&kbdev->keep_gpu_powered_count);
nr_keep_gpu_powered_ctxs > 0 ;
--nr_keep_gpu_powered_ctxs) {
kbase_pm_context_active(kbdev);
}
#if SLSI_INTEGRATION
if (kbdev->hwcnt.prev_mm) {
mutex_lock(&kbdev->hwcnt.mlock);
if ((kbdev->hwcnt.enable_for_gpr == FALSE) && (kbdev->hwcnt.s_enable_for_utilization))
kbdev->hwcnt.enable_for_utilization = TRUE;
else
kbdev->hwcnt.enable_for_utilization = FALSE;
kbase_pm_policy_change(kbdev, 2);
mutex_unlock(&kbdev->hwcnt.mlock);
} else
#endif
/* Re-enable instrumentation, if it was previously disabled */
kbase_instr_hwcnt_resume(kbdev);
/* Resume any blocked atoms (which may cause contexts to be scheduled in
* and dependent atoms to run) */
kbase_resume_suspended_soft_jobs(kbdev);
/* Resume the Job Scheduler and associated components, and start running
* atoms */
kbasep_js_resume(kbdev);
/* Matching idle call, to power off the GPU/cores if we didn't actually
* need it and the policy doesn't want it on */
kbase_pm_context_idle(kbdev);
}
mali_error kbase_pm_powerup(kbase_device *kbdev)
{
unsigned long flags;
mali_error ret;
KBASE_DEBUG_ASSERT(kbdev != NULL);
mutex_lock(&kbdev->pm.lock);
/* A suspend won't happen during startup/insmod */
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
/* Power up the GPU, don't enable IRQs as we are not ready to receive them. */
ret = kbase_pm_init_hw(kbdev, MALI_FALSE );
if (ret != MALI_ERROR_NONE) {
mutex_unlock(&kbdev->pm.lock);
return ret;
}
kbasep_pm_read_present_cores(kbdev);
kbdev->pm.debug_core_mask = kbdev->shader_present_bitmap;
/* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
kbdev->pm.active_count = 1;
spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* Ensure cycle counter is off */
kbdev->pm.gpu_cycle_counter_requests = 0;
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* We are ready to receive IRQ's now as power policy is set up, so enable them now. */
#ifdef CONFIG_MALI_DEBUG
spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
kbdev->pm.driver_ready_for_irqs = MALI_TRUE;
spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
#endif
kbase_pm_enable_interrupts(kbdev);
/* Turn on the GPU and any cores needed by the policy */
kbase_pm_do_poweron(kbdev);
mutex_unlock(&kbdev->pm.lock);
/* Idle the GPU and/or cores, if the policy wants it to */
kbase_pm_context_idle(kbdev);
return MALI_ERROR_NONE;
}
mali_error kbase_pm_powerup(kbase_device *kbdev)
{
unsigned long flags;
mali_error ret;
OSK_ASSERT(kbdev != NULL);
ret = kbase_pm_init_hw(kbdev);
if (ret != MALI_ERROR_NONE)
{
return ret;
}
kbase_pm_power_transitioning(kbdev);
kbasep_pm_read_present_cores(kbdev);
/* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
spin_lock_irqsave(&kbdev->pm.active_count_lock, flags);
kbdev->pm.active_count = 1;
spin_unlock_irqrestore(&kbdev->pm.active_count_lock, flags);
spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* Ensure cycle counter is off */
kbdev->pm.gpu_cycle_counter_requests = 0;
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
atomic_set(&kbdev->pm.pending_events, 0);
atomic_set(&kbdev->pm.work_active, KBASE_PM_WORK_ACTIVE_STATE_INACTIVE);
kbdev->pm.new_policy = NULL;
kbdev->pm.current_policy = policy_list[0];
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->init(kbdev);
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_POLICY_INIT);
/* Idle the GPU */
kbase_pm_context_idle(kbdev);
return MALI_ERROR_NONE;
}
int kbase_instr_hwcnt_disable(struct kbase_context *kctx)
{
int err = -EINVAL;
struct kbase_device *kbdev = kctx->kbdev;
err = kbase_instr_hwcnt_disable_internal(kctx);
if (err)
goto out;
/* Release the context. This had its own Power Manager Active reference
*/
kbasep_js_release_privileged_ctx(kbdev, kctx);
/* Also release our Power Manager Active reference */
kbase_pm_context_idle(kbdev);
dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p",
kctx);
out:
return err;
}
void kbase_pm_resume(struct kbase_device *kbdev)
{
/* MUST happen before any pm_context_active calls occur */
kbase_hwaccess_pm_resume(kbdev);
/* Initial active call, to power on the GPU/cores if needed */
kbase_pm_context_active(kbdev);
/* Re-enable instrumentation, if it was previously disabled */
kbase_instr_hwcnt_resume(kbdev);
/* Resume any blocked atoms (which may cause contexts to be scheduled in
* and dependent atoms to run) */
kbase_resume_suspended_soft_jobs(kbdev);
/* Resume the Job Scheduler and associated components, and start running
* atoms */
kbasep_js_resume(kbdev);
/* Matching idle call, to power off the GPU/cores if we didn't actually
* need it and the policy doesn't want it on */
kbase_pm_context_idle(kbdev);
}
void page_fault_worker(struct work_struct *data)
{
u64 fault_pfn;
u32 fault_status;
size_t new_pages;
size_t fault_rel_pfn;
struct kbase_as *faulting_as;
int as_no;
struct kbase_context *kctx;
struct kbase_device *kbdev;
struct kbase_va_region *region;
int err;
bool grown = false;
faulting_as = container_of(data, struct kbase_as, work_pagefault);
fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
as_no = faulting_as->number;
kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
/* Grab the context that was already refcounted in kbase_mmu_interrupt().
* Therefore, it cannot be scheduled out of this AS until we explicitly release it
*
* NOTE: NULL can be returned here if we're gracefully handling a spurious interrupt */
kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
if (kctx == NULL) {
/* Only handle this if not already suspended */
if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
/* Address space has no context, terminate the work */
/* AS transaction begin */
mutex_lock(&faulting_as->transaction_mutex);
kbase_mmu_disable_as(kbdev, as_no);
mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_pm_context_idle(kbdev);
}
atomic_dec(&kbdev->faults_pending);
return;
}
KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev);
fault_status = faulting_as->fault_status;
switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) {
case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT:
/* need to check against the region to handle this one */
break;
case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Permission failure");
goto fault_done;
case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Tranlation table bus fault");
goto fault_done;
case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG:
/* nothing to do, but we don't expect this fault currently */
dev_warn(kbdev->dev, "Access flag unexpectedly set");
goto fault_done;
default:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Unknown fault code");
goto fault_done;
}
/* so we have a translation fault, let's see if it is for growable
* memory */
kbase_gpu_vm_lock(kctx);
region = kbase_region_tracker_find_region_enclosing_address(kctx,
faulting_as->fault_addr);
if (!region || region->flags & KBASE_REG_FREE) {
kbase_gpu_vm_unlock(kctx);
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Memory is not mapped on the GPU");
goto fault_done;
}
if ((region->flags & GROWABLE_FLAGS_REQUIRED)
!= GROWABLE_FLAGS_REQUIRED) {
kbase_gpu_vm_unlock(kctx);
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Memory is not growable");
goto fault_done;
}
/* find the size we need to grow it by */
/* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address
* validating the fault_adress to be within a size_t from the start_pfn */
fault_rel_pfn = fault_pfn - region->start_pfn;
if (fault_rel_pfn < kbase_reg_current_backed_size(region)) {
dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring",
faulting_as->fault_addr, region->start_pfn,
region->start_pfn +
kbase_reg_current_backed_size(region));
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
/* [1] in case another page fault occurred while we were
* handling the (duplicate) page fault we need to ensure we
* don't loose the other page fault as result of us clearing
* the MMU IRQ. Therefore, after we clear the MMU IRQ we send
* an UNLOCK command that will retry any stalled memory
* transaction (which should cause the other page fault to be
* raised again).
*/
kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
AS_COMMAND_UNLOCK, 1);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_gpu_vm_unlock(kctx);
goto fault_done;
}
new_pages = make_multiple(fault_rel_pfn -
kbase_reg_current_backed_size(region) + 1,
region->extent);
/* cap to max vsize */
if (new_pages + kbase_reg_current_backed_size(region) >
region->nr_pages)
new_pages = region->nr_pages -
kbase_reg_current_backed_size(region);
if (0 == new_pages) {
/* Duplicate of a fault we've already handled, nothing to do */
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
/* See comment [1] about UNLOCK usage */
kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
AS_COMMAND_UNLOCK, 1);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_gpu_vm_unlock(kctx);
goto fault_done;
}
if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) {
if (region->gpu_alloc != region->cpu_alloc) {
if (kbase_alloc_phy_pages_helper(
region->cpu_alloc, new_pages) == 0) {
grown = true;
} else {
kbase_free_phy_pages_helper(region->gpu_alloc,
new_pages);
}
} else {
grown = true;
}
}
if (grown) {
u32 op;
/* alloc success */
KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages);
/* AS transaction begin */
mutex_lock(&faulting_as->transaction_mutex);
/* set up the new pages */
err = kbase_mmu_insert_pages(kctx, region->start_pfn + kbase_reg_current_backed_size(region) - new_pages, &kbase_get_gpu_phy_pages(region)[kbase_reg_current_backed_size(region) - new_pages], new_pages, region->flags);
if (err) {
/* failed to insert pages, handle as a normal PF */
mutex_unlock(&faulting_as->transaction_mutex);
kbase_free_phy_pages_helper(region->gpu_alloc, new_pages);
if (region->gpu_alloc != region->cpu_alloc)
kbase_free_phy_pages_helper(region->cpu_alloc,
new_pages);
kbase_gpu_vm_unlock(kctx);
/* The locked VA region will be unlocked and the cache invalidated in here */
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Page table update failure");
goto fault_done;
}
#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
#endif
#if defined(CONFIG_MALI_MIPE_ENABLED)
kbase_tlstream_aux_pagefault(
as_no,
atomic_read(&kctx->used_pages));
#endif
/* flush L2 and unlock the VA (resumes the MMU) */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
op = AS_COMMAND_FLUSH;
else
op = AS_COMMAND_FLUSH_PT;
/* clear MMU interrupt - this needs to be done after updating
* the page tables but before issuing a FLUSH command. The
* FLUSH cmd has a side effect that it restarts stalled memory
* transactions in other address spaces which may cause
* another fault to occur. If we didn't clear the interrupt at
* this stage a new IRQ might not be raised when the GPU finds
* a MMU IRQ is already pending.
*/
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx,
faulting_as->fault_addr >> PAGE_SHIFT,
new_pages,
op, 1);
mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
/* reenable this in the mask */
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_gpu_vm_unlock(kctx);
} else {
static base_jd_event_code kbase_dump_cpu_gpu_time(kbase_jd_atom *katom)
{
kbase_va_region *reg;
osk_phy_addr addr;
u64 pfn;
u32 offset;
char *page;
struct timespec ts;
base_dump_cpu_gpu_counters data;
u64 system_time;
u64 cycle_counter;
mali_addr64 jc = katom->jc;
kbase_context *kctx = katom->kctx;
u32 hi1, hi2;
memset(&data, 0, sizeof(data));
kbase_pm_context_active(kctx->kbdev);
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
do {
hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
cycle_counter = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
cycle_counter |= (((u64)hi1) << 32);
} while (hi1 != hi2);
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
do {
hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
system_time = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_LO), NULL);
hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
system_time |= (((u64)hi1) << 32);
} while (hi1 != hi2);
/* Record the CPU's idea of current time */
getnstimeofday(&ts);
kbase_pm_context_idle(kctx->kbdev);
data.sec = ts.tv_sec;
data.usec = ts.tv_nsec / 1000;
data.system_time = system_time;
data.cycle_counter = cycle_counter;
pfn = jc >> 12;
offset = jc & 0xFFF;
if (offset > 0x1000-sizeof(data))
{
/* Wouldn't fit in the page */
return BASE_JD_EVENT_JOB_CANCELLED;
}
reg = kbase_region_tracker_find_region_enclosing_address(kctx, jc);
if (!reg)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
if (! (reg->flags & KBASE_REG_GPU_WR) )
{
/* Region is not writable by GPU so we won't write to it either */
return BASE_JD_EVENT_JOB_CANCELLED;
}
if (!reg->phy_pages)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
addr = reg->phy_pages[pfn - reg->start_pfn];
if (!addr)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
page = osk_kmap(addr);
if (!page)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
memcpy(page+offset, &data, sizeof(data));
osk_sync_to_cpu(addr+offset, page+offset, sizeof(data));
osk_kunmap(addr, page);
return BASE_JD_EVENT_DONE;
}
STATIC mali_error kbase_instr_hwcnt_enable_internal(kbase_device *kbdev, kbase_context *kctx, kbase_uk_hwcnt_setup *setup)
{
unsigned long flags, pm_flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
kbasep_js_device_data *js_devdata;
u32 irq_mask;
int ret;
u64 shader_cores_needed;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_DEBUG_ASSERT(NULL != setup);
KBASE_DEBUG_ASSERT(NULL == kbdev->hwcnt.suspended_kctx);
shader_cores_needed = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER);
js_devdata = &kbdev->js_data;
/* alignment failure */
if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1)))
goto out_err;
/* Override core availability policy to ensure all cores are available */
kbase_pm_ca_instr_enable(kbdev);
/* Mark the context as active so the GPU is kept turned on */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
/* Request the cores early on synchronously - we'll release them on any errors
* (e.g. instrumentation already active) */
kbase_pm_request_cores_sync(kbdev, MALI_TRUE, shader_cores_needed);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is already enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out_unrequest_cores;
}
/* Enable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | PRFCNT_SAMPLE_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* In use, this context is the owner */
kbdev->hwcnt.kctx = kctx;
/* Remember the dump address so we can reprogram it later */
kbdev->hwcnt.addr = setup->dump_buffer;
/* Remember all the settings for suspend/resume */
if (&kbdev->hwcnt.suspended_state != setup)
memcpy(&kbdev->hwcnt.suspended_state, setup, sizeof(kbdev->hwcnt.suspended_state));
/* Request the clean */
kbdev->hwcnt.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
kbdev->hwcnt.triggered = 0;
/* Clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
ret = queue_work(kbdev->hwcnt.cache_clean_wq, &kbdev->hwcnt.cache_clean_work);
KBASE_DEBUG_ASSERT(ret);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Wait for cacheclean to complete */
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE);
/* Schedule the context in */
kbasep_js_schedule_privileged_ctx(kbdev, kctx);
/* Configure */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_OFF, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), setup->dump_buffer & 0xFFFFFFFF, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), setup->dump_buffer >> 32, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), setup->jm_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), setup->shader_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), setup->l3_cache_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), setup->mmu_l2_bm, kctx);
/* Due to PRLAM-8186 we need to disable the Tiler before we enable the HW counter dump. */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, kctx);
else
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, kctx);
/* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.triggered = 1;
wake_up(&kbdev->hwcnt.wait);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
err = MALI_ERROR_NONE;
dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx);
return err;
out_unrequest_cores:
kbase_pm_unrequest_cores(kbdev, MALI_TRUE, shader_cores_needed);
kbase_pm_context_idle(kbdev);
out_err:
return err;
}
/**
* @brief Disable HW counters collection
*
* Note: might sleep, waiting for an ongoing dump to complete
*/
mali_error kbase_instr_hwcnt_disable(kbase_context *kctx)
{
unsigned long flags, pm_flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
u32 irq_mask;
kbase_device *kbdev;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
while (1) {
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is not enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out;
}
if (kbdev->hwcnt.kctx != kctx) {
/* Instrumentation has been setup for another context */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out;
}
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE)
break;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Ongoing dump/setup - wait for its completion */
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
}
kbdev->hwcnt.state = KBASE_INSTR_STATE_DISABLED;
kbdev->hwcnt.triggered = 0;
/* Disable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* Disable the counters */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx);
kbdev->hwcnt.kctx = NULL;
kbdev->hwcnt.addr = 0ULL;
kbase_pm_ca_instr_disable(kbdev);
kbase_pm_unrequest_cores(kbdev, MALI_TRUE, kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER));
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Release the context. This had its own Power Manager Active reference */
kbasep_js_release_privileged_ctx(kbdev, kctx);
/* Also release our Power Manager Active reference */
kbase_pm_context_idle(kbdev);
dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
err = MALI_ERROR_NONE;
out:
return err;
}
/**
* kbase_destroy_context - Destroy a kernel base context.
* @kctx: Context to destroy
*
* Calls kbase_destroy_os_context() to free OS specific structures.
* Will release all outstanding regions.
*/
void kbase_destroy_context(struct kbase_context *kctx)
{
struct kbase_device *kbdev;
int pages;
unsigned long pending_regions_to_clean;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u);
/* Ensure the core is powered up for the destroy process */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
kbase_jd_zap_context(kctx);
kbase_event_cleanup(kctx);
kbase_gpu_vm_lock(kctx);
/* MMU is disabled as part of scheduling out the context */
kbase_mmu_free_pgd(kctx);
/* drop the aliasing sink page now that it can't be mapped anymore */
kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false);
/* free pending region setups */
pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK;
while (pending_regions_to_clean) {
unsigned int cookie = __ffs(pending_regions_to_clean);
BUG_ON(!kctx->pending_regions[cookie]);
kbase_reg_pending_dtor(kctx->pending_regions[cookie]);
kctx->pending_regions[cookie] = NULL;
pending_regions_to_clean &= ~(1UL << cookie);
}
kbase_region_tracker_term(kctx);
kbase_gpu_vm_unlock(kctx);
/* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */
kbasep_js_kctx_term(kctx);
kbase_jd_exit(kctx);
kbase_pm_context_idle(kbdev);
kbase_mmu_term(kctx);
pages = atomic_read(&kctx->used_pages);
if (pages != 0)
dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
kbase_mem_pool_term(&kctx->mem_pool);
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
vfree(kctx);
}
void kbasep_js_runpool_release_ctx( kbase_device *kbdev, kbase_context *kctx )
{
kbasep_js_device_data *js_devdata;
kbasep_js_kctx_info *js_kctx_info;
kbasep_js_policy *js_policy;
kbasep_js_per_as_data *js_per_as_data;
mali_bool was_descheduled = MALI_FALSE;
int saved_as_nr;
kbase_as *current_as;
int new_ref_count;
mali_bool nss_state_changed = MALI_FALSE;
OSK_ASSERT( kbdev != NULL );
OSK_ASSERT( kctx != NULL );
js_kctx_info = &kctx->jctx.sched_info;
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
osk_mutex_lock( &js_kctx_info->ctx.jsctx_mutex );
osk_mutex_lock( &js_devdata->runpool_mutex );
/* Ensure context really is scheduled in */
OSK_ASSERT( js_kctx_info->ctx.is_scheduled != MALI_FALSE );
/* The saved_as_nr must be accessed under lock, but we also need to take a
* sleeping mutex. Since the ctx is known to be busy-refcounted, we can
* just take the runpool lock briefly, then taken it again later (the as_nr
* won't be reassigned due to being busy).
*
* We ASSERT on this fact */
osk_spinlock_irq_lock( &js_devdata->runpool_irq.lock );
{
saved_as_nr = kctx->as_nr;
OSK_ASSERT( saved_as_nr != KBASEP_AS_NR_INVALID );
js_per_as_data = &js_devdata->runpool_irq.per_as_data[saved_as_nr];
OSK_ASSERT( js_per_as_data->as_busy_refcount > 0 );
}
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
/*
* Transaction begins on AS and runpool_irq
*
* Doubly-assert that our previous facts are still true
*/
current_as = &kbdev->as[saved_as_nr];
osk_mutex_lock( ¤t_as->transaction_mutex );
osk_spinlock_irq_lock( &js_devdata->runpool_irq.lock );
OSK_ASSERT( saved_as_nr == kctx->as_nr );
OSK_ASSERT( js_per_as_data->as_busy_refcount > 0 );
/* Update refcount */
new_ref_count = --(js_per_as_data->as_busy_refcount);
/* Make a set of checks to see if the context should be scheduled out */
if ( new_ref_count == 0
&& ( kctx->jctx.sched_info.ctx.nr_jobs == 0
|| kbasep_js_is_submit_allowed( js_devdata, kctx ) == MALI_FALSE ) )
{
/* Last reference, and we've been told to remove this context from the Run Pool */
OSK_PRINT_INFO(OSK_BASE_JM, "JS: RunPool Remove Context %p because as_busy_refcount=%d, jobs=%d, allowed=%d",
kctx,
new_ref_count,
js_kctx_info->ctx.nr_jobs,
kbasep_js_is_submit_allowed( js_devdata, kctx ) );
kbasep_js_policy_runpool_remove_ctx( js_policy, kctx );
/* Stop any more refcounts occuring on the context */
js_per_as_data->kctx = NULL;
/* Ensure we prevent the context from submitting any new jobs
* e.g. from kbasep_js_try_run_next_job_on_slot_irq_nolock() */
kbasep_js_clear_submit_allowed( js_devdata, kctx );
/* Disable the MMU on the affected address space, and indicate it's invalid */
kbase_mmu_disable( kctx );
kctx->as_nr = KBASEP_AS_NR_INVALID;
/* NSS handling */
nss_state_changed = kbasep_js_check_and_deref_nss_running_ctx( js_devdata, kctx );
/*
* Transaction ends on AS and runpool_irq:
*
* By this point, the AS-related data is now clear and ready for re-use.
*
* Since releases only occur once for each previous successful retain, and no more
* retains are allowed on this context, no other thread will be operating in this
* code whilst we are
*/
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
osk_mutex_unlock( ¤t_as->transaction_mutex );
/* Free up the address space */
js_devdata->as_free |= ((u16)(1u << saved_as_nr));
/* Note: Don't reuse saved_as_nr now */
/* update book-keeping info */
--(js_devdata->nr_contexts_running);
js_kctx_info->ctx.is_scheduled = MALI_FALSE;
/* Signal any waiter that the context is not scheduled, so is safe for
* termination - once the jsctx_mutex is also dropped, and jobs have
* finished. */
osk_waitq_set( &js_kctx_info->ctx.not_scheduled_waitq );
/* Handle dying contexts */
if ( js_kctx_info->ctx.is_dying != MALI_FALSE )
{
/* This happens asynchronously */
OSK_PRINT_INFO(OSK_BASE_JM, "JS: ** Killing Context %p on RunPool Remove **", kctx );
kbasep_js_policy_kill_all_ctx_jobs( js_policy, kctx );
}
/* Queue an action to occur after we've dropped the lock */
was_descheduled = MALI_TRUE;
}
else
{
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
osk_mutex_unlock( ¤t_as->transaction_mutex );
}
osk_mutex_unlock( &js_devdata->runpool_mutex );
/* Do we have an action queued whilst the lock was held? */
if ( was_descheduled != MALI_FALSE )
{
/* Determine whether this context should be requeued on the policy queue */
if ( js_kctx_info->ctx.nr_jobs > 0 && js_kctx_info->ctx.is_dying == MALI_FALSE )
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Requeue Context %p", kctx );
osk_mutex_lock( &js_devdata->queue_mutex );
kbasep_js_policy_enqueue_ctx( js_policy, kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
}
else
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Idling Context %p (not requeued)", kctx );
/* Notify PM that a context has gone idle */
kbase_pm_context_idle(kctx->kbdev);
}
}
/* We've finished with this context for now, so drop the lock for it. */
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
if ( was_descheduled != MALI_FALSE )
{
/* We've freed up an address space, so let's try to schedule in another
* context
*
* Note: if there's a context to schedule in, then it also tries to run
* another job, in case the new context has jobs satisfying requirements
* that no other context/job in the runpool does */
kbasep_js_try_schedule_head_ctx( kbdev );
}
if ( nss_state_changed != MALI_FALSE )
{
osk_mutex_lock( &js_devdata->runpool_mutex );
kbasep_js_try_run_next_job( kbdev );
osk_mutex_unlock( &js_devdata->runpool_mutex );
}
}
void kbasep_js_try_schedule_head_ctx( kbase_device *kbdev )
{
kbasep_js_device_data *js_devdata;
mali_bool has_kctx;
kbase_context *head_kctx;
kbasep_js_kctx_info *js_kctx_info;
mali_bool is_runpool_full;
OSK_ASSERT( kbdev != NULL );
js_devdata = &kbdev->js_data;
/* Make a speculative check on the Run Pool - this MUST be repeated once
* we've obtained a context from the queue and reobtained the Run Pool
* lock */
osk_mutex_lock( &js_devdata->runpool_mutex );
is_runpool_full = (mali_bool)( js_devdata->nr_contexts_running >= kbdev->nr_address_spaces );
osk_mutex_unlock( &js_devdata->runpool_mutex );
if ( is_runpool_full != MALI_FALSE )
{
/* No free address spaces - nothing to do */
return;
}
/* Grab the context off head of queue - if there is one */
osk_mutex_lock( &js_devdata->queue_mutex );
has_kctx = kbasep_js_policy_dequeue_head_ctx( &js_devdata->policy, &head_kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
if ( has_kctx == MALI_FALSE )
{
/* No ctxs to run - nothing to do */
return;
}
js_kctx_info = &head_kctx->jctx.sched_info;
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Dequeue Context %p", head_kctx );
/*
* Atomic transaction on the Context and Run Pool begins
*/
osk_mutex_lock( &js_kctx_info->ctx.jsctx_mutex );
osk_mutex_lock( &js_devdata->runpool_mutex );
/* Re-check to see if the Run Pool is full */
is_runpool_full = (mali_bool)( js_devdata->nr_contexts_running >= kbdev->nr_address_spaces );
if ( is_runpool_full != MALI_FALSE )
{
/* No free address spaces - roll back the transaction so far and return */
osk_mutex_unlock( &js_devdata->runpool_mutex );
/* Only requeue if not dying - which might occur through zapping-whilst-scheduling */
if ( js_kctx_info->ctx.is_dying == MALI_FALSE )
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Transaction rollback: Requeue Context %p", head_kctx );
osk_mutex_lock( &js_devdata->queue_mutex );
kbasep_js_policy_enqueue_ctx( &js_devdata->policy, head_kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
}
else
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Transaction rollback: Context %p is dying. Kill remaining jobs and pm-idle ctx", head_kctx );
OSK_ASSERT( js_kctx_info->ctx.nr_jobs > 0 );
/* Notify PM that a context has gone idle */
kbase_pm_context_idle(kbdev);
/* Kill all the jobs present (call kbase_jd_cancel on all jobs) */
kbasep_js_policy_kill_all_ctx_jobs( &js_devdata->policy, head_kctx );
/* Nothing more to be done to kill the context here, kbase_jd_zap_context
* waits for all jobs to be cancelled */
}
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
return;
}
OSK_PRINT_INFO(OSK_BASE_JM, "JS: RunPool Add Context %p", head_kctx );
/* update book-keeping info */
js_kctx_info->ctx.is_scheduled = MALI_TRUE;
++(js_devdata->nr_contexts_running);
/* Cause any future waiter-on-termination to wait until the context is
* descheduled */
osk_waitq_clear( &js_kctx_info->ctx.not_scheduled_waitq );
/* Do all the necessaries to pick the address space (inc. update book-keeping info)
* Add the context to the Run Pool, and allow it to run jobs */
assign_and_activate_kctx_addr_space( kbdev, head_kctx );
/* Check and setup HW counters dumping */
osk_spinlock_lock(&kbdev->hwcnt_lock);
osk_spinlock_irq_lock(&js_devdata->runpool_irq.lock);
if (head_kctx == kbdev->hwcnt_context &&
kbdev->hwcnt_is_setup == MALI_FALSE)
{
/* Setup the base address */
#if BASE_HW_ISSUE_8186
u32 val;
/* Save and clear PRFCNT_TILER_EN */
val = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), head_kctx);
if(0 != val)
{
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, head_kctx);
}
/* Update PRFCNT_CONFIG with TILER_EN = 0 */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (head_kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, head_kctx);
/* Restore PRFCNT_TILER_EN */
if(0 != val)
{
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), val, head_kctx);
}
#else
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (head_kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, head_kctx);
#endif
/* Prevent the context to be scheduled out */
kbasep_js_runpool_retain_ctx_nolock(kbdev, head_kctx);
kbdev->hwcnt_is_setup = MALI_TRUE;
}
osk_spinlock_irq_unlock(&js_devdata->runpool_irq.lock);
osk_spinlock_unlock(&kbdev->hwcnt_lock);
/* Try to run the next job, in case this context has jobs that match the
* job slot requirements, but none of the other currently running contexts
* do */
kbasep_js_try_run_next_job( kbdev );
/* Transaction complete */
osk_mutex_unlock( &js_devdata->runpool_mutex );
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
/* Note: after this point, the context could potentially get scheduled out immediately */
}