STATIC INLINE mali_bool kbase_js_affinity_is_violating(kbase_device *kbdev, u64 *affinities)
{
/* This implementation checks whether the two slots involved in Generic thread creation
* have intersecting affinity. This is due to micro-architectural issues where a job in
* slot A targetting cores used by slot B could prevent the job in slot B from making
* progress until the job in slot A has completed.
*
* @note It just so happens that this restriction also allows
* BASE_HW_ISSUE_8987 to be worked around by placing on job slot 2 the
* atoms from ctxs with KBASE_CTX_FLAG_HINT_ONLY_COMPUTE flag set
*/
u64 affinity_set_left;
u64 affinity_set_right;
u64 intersection;
KBASE_DEBUG_ASSERT(affinities != NULL);
affinity_set_left = affinities[1];
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
/* The left set also includes those on the Fragment slot when
* we are using the HW workaround for BASE_HW_ISSUE_8987 */
affinity_set_left |= affinities[0];
}
affinity_set_right = affinities[2];
/* A violation occurs when any bit in the left_set is also in the right_set */
intersection = affinity_set_left & affinity_set_right;
return (mali_bool) (intersection != (u64) 0u);
}
/*
* Hold the runpool_mutex for this
*/
static inline bool timer_callback_should_run(struct kbase_device *kbdev)
{
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
s8 nr_running_ctxs;
lockdep_assert_held(&kbdev->js_data.runpool_mutex);
/* Timer must stop if we are suspending */
if (backend->suspend_timer)
return false;
/* nr_contexts_pullable is updated with the runpool_mutex. However, the
* locking in the caller gives us a barrier that ensures
* nr_contexts_pullable is up-to-date for reading */
nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable);
#ifdef CONFIG_MALI_DEBUG
if (kbdev->js_data.softstop_always) {
/* Debug support for allowing soft-stop on a single context */
return true;
}
#endif /* CONFIG_MALI_DEBUG */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
/* Timeouts would have to be 4x longer (due to micro-
* architectural design) to support OpenCL conformance tests, so
* only run the timer when there's:
* - 2 or more CL contexts
* - 1 or more GLES contexts
*
* NOTE: We will treat a context that has both Compute and Non-
* Compute jobs will be treated as an OpenCL context (hence, we
* don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE).
*/
{
s8 nr_compute_ctxs =
kbasep_js_ctx_attr_count_on_runpool(kbdev,
KBASEP_JS_CTX_ATTR_COMPUTE);
s8 nr_noncompute_ctxs = nr_running_ctxs -
nr_compute_ctxs;
return (bool) (nr_compute_ctxs >= 2 ||
nr_noncompute_ctxs > 0);
}
} else {
/* Run the timer callback whenever you have at least 1 context
*/
return (bool) (nr_running_ctxs > 0);
}
}
STATIC INLINE mali_bool affinity_job_uses_high_cores(kbase_device *kbdev, kbase_jd_atom *katom)
{
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
kbase_context *kctx;
kbase_context_flags ctx_flags;
kctx = katom->kctx;
ctx_flags = kctx->jctx.sched_info.ctx.flags;
/* In this HW Workaround, compute-only jobs/contexts use the high cores
* during a core-split, all other contexts use the low cores. */
return (mali_bool) ((katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) != 0 || (ctx_flags & KBASE_CTX_FLAG_HINT_ONLY_COMPUTE) != 0);
}
return MALI_FALSE;
}
bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev,
int js)
{
/*
* Here are the reasons for using job slot 2:
* - BASE_HW_ISSUE_8987 (which is entirely used for that purpose)
* - In absence of the above, then:
* - Atoms with BASE_JD_REQ_COHERENT_GROUP
* - But, only when there aren't contexts with
* KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on
* all cores on slot 1 could be blocked by those using a coherent group
* on slot 2
* - And, only when you actually have 2 or more coregroups - if you
* only have 1 coregroup, then having jobs for slot 2 implies they'd
* also be for slot 1, meaning you'll get interference from them. Jobs
* able to run on slot 2 could also block jobs that can only run on
* slot 1 (tiler jobs)
*/
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
return true;
if (js != 2)
return true;
/* Only deal with js==2 now: */
if (kbdev->gpu_props.num_core_groups > 1) {
/* Only use slot 2 in the 2+ coregroup case */
if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev,
KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) ==
false) {
/* ...But only when we *don't* have atoms that run on
* all cores */
/* No specific check for BASE_JD_REQ_COHERENT_GROUP
* atoms - the policy will sort that out */
return true;
}
}
/* Above checks failed mean we shouldn't use slot 2 */
return false;
}
/**
* @brief Decide whether a split in core affinity is required across job slots
*
* The following locking conditions are made on the caller:
* - it must hold kbasep_js_device_data::runpool_irq::lock
*
* @param kbdev The kbase device structure of the device
* @return MALI_FALSE if a core split is not required
* @return != MALI_FALSE if a core split is required.
*/
STATIC INLINE mali_bool kbase_affinity_requires_split(kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(kbdev != NULL);
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
s8 nr_compute_ctxs = kbasep_js_ctx_attr_count_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE);
s8 nr_noncompute_ctxs = kbasep_js_ctx_attr_count_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_NON_COMPUTE);
/* In this case, a mix of Compute+Non-Compute determines whether a
* core-split is required, to ensure jobs with different numbers of RMUs
* don't use the same cores.
*
* When it's entirely compute, or entirely non-compute, then no split is
* required.
*
* A context can be both Compute and Non-compute, in which case this will
* correctly decide that a core-split is required. */
return (mali_bool) (nr_compute_ctxs > 0 && nr_noncompute_ctxs > 0);
}
return MALI_FALSE;
}
void page_fault_worker(struct work_struct *data)
{
u64 fault_pfn;
u32 fault_status;
size_t new_pages;
size_t fault_rel_pfn;
struct kbase_as *faulting_as;
int as_no;
struct kbase_context *kctx;
struct kbase_device *kbdev;
struct kbase_va_region *region;
int err;
bool grown = false;
faulting_as = container_of(data, struct kbase_as, work_pagefault);
fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
as_no = faulting_as->number;
kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
/* Grab the context that was already refcounted in kbase_mmu_interrupt().
* Therefore, it cannot be scheduled out of this AS until we explicitly release it
*
* NOTE: NULL can be returned here if we're gracefully handling a spurious interrupt */
kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
if (kctx == NULL) {
/* Only handle this if not already suspended */
if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
/* Address space has no context, terminate the work */
/* AS transaction begin */
mutex_lock(&faulting_as->transaction_mutex);
kbase_mmu_disable_as(kbdev, as_no);
mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_pm_context_idle(kbdev);
}
atomic_dec(&kbdev->faults_pending);
return;
}
KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev);
fault_status = faulting_as->fault_status;
switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) {
case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT:
/* need to check against the region to handle this one */
break;
case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Permission failure");
goto fault_done;
case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Tranlation table bus fault");
goto fault_done;
case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG:
/* nothing to do, but we don't expect this fault currently */
dev_warn(kbdev->dev, "Access flag unexpectedly set");
goto fault_done;
default:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Unknown fault code");
goto fault_done;
}
/* so we have a translation fault, let's see if it is for growable
* memory */
kbase_gpu_vm_lock(kctx);
region = kbase_region_tracker_find_region_enclosing_address(kctx,
faulting_as->fault_addr);
if (!region || region->flags & KBASE_REG_FREE) {
kbase_gpu_vm_unlock(kctx);
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Memory is not mapped on the GPU");
goto fault_done;
}
if ((region->flags & GROWABLE_FLAGS_REQUIRED)
!= GROWABLE_FLAGS_REQUIRED) {
kbase_gpu_vm_unlock(kctx);
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Memory is not growable");
goto fault_done;
}
/* find the size we need to grow it by */
/* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address
* validating the fault_adress to be within a size_t from the start_pfn */
fault_rel_pfn = fault_pfn - region->start_pfn;
if (fault_rel_pfn < kbase_reg_current_backed_size(region)) {
dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring",
faulting_as->fault_addr, region->start_pfn,
region->start_pfn +
kbase_reg_current_backed_size(region));
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
/* [1] in case another page fault occurred while we were
* handling the (duplicate) page fault we need to ensure we
* don't loose the other page fault as result of us clearing
* the MMU IRQ. Therefore, after we clear the MMU IRQ we send
* an UNLOCK command that will retry any stalled memory
* transaction (which should cause the other page fault to be
* raised again).
*/
kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
AS_COMMAND_UNLOCK, 1);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_gpu_vm_unlock(kctx);
goto fault_done;
}
new_pages = make_multiple(fault_rel_pfn -
kbase_reg_current_backed_size(region) + 1,
region->extent);
/* cap to max vsize */
if (new_pages + kbase_reg_current_backed_size(region) >
region->nr_pages)
new_pages = region->nr_pages -
kbase_reg_current_backed_size(region);
if (0 == new_pages) {
/* Duplicate of a fault we've already handled, nothing to do */
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
/* See comment [1] about UNLOCK usage */
kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
AS_COMMAND_UNLOCK, 1);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_gpu_vm_unlock(kctx);
goto fault_done;
}
if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) {
if (region->gpu_alloc != region->cpu_alloc) {
if (kbase_alloc_phy_pages_helper(
region->cpu_alloc, new_pages) == 0) {
grown = true;
} else {
kbase_free_phy_pages_helper(region->gpu_alloc,
new_pages);
}
} else {
grown = true;
}
}
if (grown) {
u32 op;
/* alloc success */
KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages);
/* AS transaction begin */
mutex_lock(&faulting_as->transaction_mutex);
/* set up the new pages */
err = kbase_mmu_insert_pages(kctx, region->start_pfn + kbase_reg_current_backed_size(region) - new_pages, &kbase_get_gpu_phy_pages(region)[kbase_reg_current_backed_size(region) - new_pages], new_pages, region->flags);
if (err) {
/* failed to insert pages, handle as a normal PF */
mutex_unlock(&faulting_as->transaction_mutex);
kbase_free_phy_pages_helper(region->gpu_alloc, new_pages);
if (region->gpu_alloc != region->cpu_alloc)
kbase_free_phy_pages_helper(region->cpu_alloc,
new_pages);
kbase_gpu_vm_unlock(kctx);
/* The locked VA region will be unlocked and the cache invalidated in here */
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Page table update failure");
goto fault_done;
}
#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
#endif
#if defined(CONFIG_MALI_MIPE_ENABLED)
kbase_tlstream_aux_pagefault(
as_no,
atomic_read(&kctx->used_pages));
#endif
/* flush L2 and unlock the VA (resumes the MMU) */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
op = AS_COMMAND_FLUSH;
else
op = AS_COMMAND_FLUSH_PT;
/* clear MMU interrupt - this needs to be done after updating
* the page tables but before issuing a FLUSH command. The
* FLUSH cmd has a side effect that it restarts stalled memory
* transactions in other address spaces which may cause
* another fault to occur. If we didn't clear the interrupt at
* this stage a new IRQ might not be raised when the GPU finds
* a MMU IRQ is already pending.
*/
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx,
faulting_as->fault_addr >> PAGE_SHIFT,
new_pages,
op, 1);
mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
/* reenable this in the mask */
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_gpu_vm_unlock(kctx);
} else {
STATIC mali_error kbase_instr_hwcnt_enable_internal(kbase_device *kbdev, kbase_context *kctx, kbase_uk_hwcnt_setup *setup)
{
unsigned long flags, pm_flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
kbasep_js_device_data *js_devdata;
u32 irq_mask;
int ret;
u64 shader_cores_needed;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_DEBUG_ASSERT(NULL != setup);
KBASE_DEBUG_ASSERT(NULL == kbdev->hwcnt.suspended_kctx);
shader_cores_needed = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER);
js_devdata = &kbdev->js_data;
/* alignment failure */
if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1)))
goto out_err;
/* Override core availability policy to ensure all cores are available */
kbase_pm_ca_instr_enable(kbdev);
/* Mark the context as active so the GPU is kept turned on */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
/* Request the cores early on synchronously - we'll release them on any errors
* (e.g. instrumentation already active) */
kbase_pm_request_cores_sync(kbdev, MALI_TRUE, shader_cores_needed);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is already enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out_unrequest_cores;
}
/* Enable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | PRFCNT_SAMPLE_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* In use, this context is the owner */
kbdev->hwcnt.kctx = kctx;
/* Remember the dump address so we can reprogram it later */
kbdev->hwcnt.addr = setup->dump_buffer;
/* Remember all the settings for suspend/resume */
if (&kbdev->hwcnt.suspended_state != setup)
memcpy(&kbdev->hwcnt.suspended_state, setup, sizeof(kbdev->hwcnt.suspended_state));
/* Request the clean */
kbdev->hwcnt.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
kbdev->hwcnt.triggered = 0;
/* Clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
ret = queue_work(kbdev->hwcnt.cache_clean_wq, &kbdev->hwcnt.cache_clean_work);
KBASE_DEBUG_ASSERT(ret);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Wait for cacheclean to complete */
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE);
/* Schedule the context in */
kbasep_js_schedule_privileged_ctx(kbdev, kctx);
/* Configure */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_OFF, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), setup->dump_buffer & 0xFFFFFFFF, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), setup->dump_buffer >> 32, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), setup->jm_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), setup->shader_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), setup->l3_cache_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), setup->mmu_l2_bm, kctx);
/* Due to PRLAM-8186 we need to disable the Tiler before we enable the HW counter dump. */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, kctx);
else
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, kctx);
/* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.triggered = 1;
wake_up(&kbdev->hwcnt.wait);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
err = MALI_ERROR_NONE;
dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx);
return err;
out_unrequest_cores:
kbase_pm_unrequest_cores(kbdev, MALI_TRUE, shader_cores_needed);
kbase_pm_context_idle(kbdev);
out_err:
return err;
}
uintptr_t kbasep_get_config_value(struct kbase_device *kbdev, const kbase_attribute *attributes, int attribute_id)
{
const kbase_attribute *attr;
KBASE_DEBUG_ASSERT(attributes != NULL);
attr = kbasep_get_next_attribute(attributes, attribute_id);
if (attr != NULL)
return attr->data;
/* default values */
switch (attribute_id) {
case KBASE_CONFIG_ATTR_GPU_IRQ_THROTTLE_TIME_US:
return DEFAULT_IRQ_THROTTLE_TIME_US;
/* Begin scheduling defaults */
case KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS:
return DEFAULT_JS_SCHEDULING_TICK_NS;
case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS:
return DEFAULT_JS_SOFT_STOP_TICKS;
case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL:
return DEFAULT_JS_SOFT_STOP_TICKS_CL;
case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS:
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
return DEFAULT_JS_HARD_STOP_TICKS_SS_HW_ISSUE_8408;
else
return DEFAULT_JS_HARD_STOP_TICKS_SS;
case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL:
return DEFAULT_JS_HARD_STOP_TICKS_CL;
case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS:
return DEFAULT_JS_HARD_STOP_TICKS_NSS;
case KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS:
return DEFAULT_JS_CTX_TIMESLICE_NS;
case KBASE_CONFIG_ATTR_JS_CFS_CTX_RUNTIME_INIT_SLICES:
return DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES;
case KBASE_CONFIG_ATTR_JS_CFS_CTX_RUNTIME_MIN_SLICES:
return DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES;
case KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS:
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
return DEFAULT_JS_RESET_TICKS_SS_HW_ISSUE_8408;
else
return DEFAULT_JS_RESET_TICKS_SS;
case KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL:
return DEFAULT_JS_RESET_TICKS_CL;
case KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS:
return DEFAULT_JS_RESET_TICKS_NSS;
case KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS:
return DEFAULT_JS_RESET_TIMEOUT_MS;
/* End scheduling defaults */
case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS:
return 0;
case KBASE_CONFIG_ATTR_PLATFORM_FUNCS:
return 0;
case KBASE_CONFIG_ATTR_SECURE_BUT_LOSS_OF_PERFORMANCE:
return DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE;
case KBASE_CONFIG_ATTR_CPU_SPEED_FUNC:
return DEFAULT_CPU_SPEED_FUNC;
case KBASE_CONFIG_ATTR_GPU_SPEED_FUNC:
return 0;
case KBASE_CONFIG_ATTR_ARID_LIMIT:
return DEFAULT_ARID_LIMIT;
case KBASE_CONFIG_ATTR_AWID_LIMIT:
return DEFAULT_AWID_LIMIT;
case KBASE_CONFIG_ATTR_ALTERNATIVE_HWC:
return DEFAULT_ALTERNATIVE_HWC;
case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_DVFS_FREQ:
return DEFAULT_PM_DVFS_FREQ;
case KBASE_CONFIG_ATTR_PM_SHADER_POWEROFF_TIME:
return DEFAULT_PM_SHADER_POWEROFF_TIME;
default:
KBASE_DEBUG_PRINT_ERROR(KBASE_CORE, "kbasep_get_config_value. Cannot get value of attribute with id=%d and no default value defined", attribute_id);
return 0;
}
}
static enum hrtimer_restart timer_callback(struct hrtimer *timer)
{
unsigned long flags;
struct kbase_device *kbdev;
struct kbasep_js_device_data *js_devdata;
struct kbase_backend_data *backend;
int s;
bool reset_needed = false;
KBASE_DEBUG_ASSERT(timer != NULL);
backend = container_of(timer, struct kbase_backend_data,
scheduling_timer);
kbdev = container_of(backend, struct kbase_device, hwaccess.backend);
js_devdata = &kbdev->js_data;
/* Loop through the slots */
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) {
struct kbase_jd_atom *atom = NULL;
if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) {
atom = kbase_gpu_inspect(kbdev, s, 0);
KBASE_DEBUG_ASSERT(atom != NULL);
}
if (atom != NULL) {
/* The current version of the model doesn't support
* Soft-Stop */
if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
u32 ticks = atom->sched_info.cfs.ticks++;
#if !CINSTR_DUMPING_ENABLED
u32 soft_stop_ticks, hard_stop_ticks,
gpu_reset_ticks;
if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
soft_stop_ticks =
js_devdata->soft_stop_ticks_cl;
hard_stop_ticks =
js_devdata->hard_stop_ticks_cl;
gpu_reset_ticks =
js_devdata->gpu_reset_ticks_cl;
} else {
soft_stop_ticks =
js_devdata->soft_stop_ticks;
hard_stop_ticks =
js_devdata->hard_stop_ticks_ss;
gpu_reset_ticks =
js_devdata->gpu_reset_ticks_ss;
}
/* Job is Soft-Stoppable */
if (ticks == soft_stop_ticks) {
int disjoint_threshold =
KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD;
u32 softstop_flags = 0u;
/* Job has been scheduled for at least
* js_devdata->soft_stop_ticks ticks.
* Soft stop the slot so we can run
* other jobs.
*/
dev_dbg(kbdev->dev, "Soft-stop");
#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS
/* nr_user_contexts_running is updated
* with the runpool_mutex, but we can't
* take that here.
*
* However, if it's about to be
* increased then the new context can't
* run any jobs until they take the
* runpool_irq lock, so it's OK to
* observe the older value.
*
* Similarly, if it's about to be
* decreased, the last job from another
* context has already finished, so it's
* not too bad that we observe the older
* value and register a disjoint event
* when we try soft-stopping */
if (js_devdata->nr_user_contexts_running
>= disjoint_threshold)
softstop_flags |=
JS_COMMAND_SW_CAUSES_DISJOINT;
kbase_job_slot_softstop_swflags(kbdev,
s, atom, softstop_flags);
#endif
} else if (ticks == hard_stop_ticks) {
/* Job has been scheduled for at least
* js_devdata->hard_stop_ticks_ss ticks.
* It should have been soft-stopped by
* now. Hard stop the slot.
*/
#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
int ms =
js_devdata->scheduling_period_ns
/ 1000000u;
dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
(unsigned long)ticks,
(unsigned long)ms);
kbase_job_slot_hardstop(atom->kctx, s,
atom);
#endif
} else if (ticks == gpu_reset_ticks) {
/* Job has been scheduled for at least
* js_devdata->gpu_reset_ticks_ss ticks.
* It should have left the GPU by now.
* Signal that the GPU needs to be
* reset.
*/
reset_needed = true;
}
#else /* !CINSTR_DUMPING_ENABLED */
/* NOTE: During CINSTR_DUMPING_ENABLED, we use
* the alternate timeouts, which makes the hard-
* stop and GPU reset timeout much longer. We
* also ensure that we don't soft-stop at all.
*/
if (ticks == js_devdata->soft_stop_ticks) {
/* Job has been scheduled for at least
* js_devdata->soft_stop_ticks. We do
* not soft-stop during
* CINSTR_DUMPING_ENABLED, however.
*/
dev_dbg(kbdev->dev, "Soft-stop");
} else if (ticks ==
js_devdata->hard_stop_ticks_dumping) {
/* Job has been scheduled for at least
* js_devdata->hard_stop_ticks_dumping
* ticks. Hard stop the slot.
*/
#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
int ms =
js_devdata->scheduling_period_ns
/ 1000000u;
dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
(unsigned long)ticks,
(unsigned long)ms);
kbase_job_slot_hardstop(atom->kctx, s,
atom);
#endif
} else if (ticks ==
js_devdata->gpu_reset_ticks_dumping) {
/* Job has been scheduled for at least
* js_devdata->gpu_reset_ticks_dumping
* ticks. It should have left the GPU by
* now. Signal that the GPU needs to be
* reset.
*/
reset_needed = true;
}
#endif /* !CINSTR_DUMPING_ENABLED */
}
}
}
#if KBASE_GPU_RESET_EN
if (reset_needed) {
dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve.");
if (kbase_prepare_to_reset_gpu_locked(kbdev))
kbase_reset_gpu_locked(kbdev);
}
#endif /* KBASE_GPU_RESET_EN */
/* the timer is re-issued if there is contexts in the run-pool */
if (backend->timer_running)
hrtimer_start(&backend->scheduling_timer,
HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
HRTIMER_MODE_REL);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return HRTIMER_NORESTART;
}
uintptr_t kbasep_get_config_value(struct kbase_device *kbdev, const struct kbase_attribute *attributes, int attribute_id)
{
const struct kbase_attribute *attr;
KBASE_DEBUG_ASSERT(attributes != NULL);
attr = kbasep_get_next_attribute(attributes, attribute_id);
if (attr != NULL)
return attr->data;
/* default values */
switch (attribute_id) {
/* Begin scheduling defaults */
case KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS:
return DEFAULT_JS_SCHEDULING_TICK_NS;
case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS:
return DEFAULT_JS_SOFT_STOP_TICKS;
case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL:
return DEFAULT_JS_SOFT_STOP_TICKS_CL;
case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS:
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
return DEFAULT_JS_HARD_STOP_TICKS_SS_HW_ISSUE_8408;
else
return DEFAULT_JS_HARD_STOP_TICKS_SS;
case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL:
return DEFAULT_JS_HARD_STOP_TICKS_CL;
case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS:
return DEFAULT_JS_HARD_STOP_TICKS_NSS;
case KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS:
return DEFAULT_JS_CTX_TIMESLICE_NS;
case KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS:
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
return DEFAULT_JS_RESET_TICKS_SS_HW_ISSUE_8408;
else
return DEFAULT_JS_RESET_TICKS_SS;
case KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL:
return DEFAULT_JS_RESET_TICKS_CL;
case KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS:
return DEFAULT_JS_RESET_TICKS_NSS;
case KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS:
return DEFAULT_JS_RESET_TIMEOUT_MS;
/* End scheduling defaults */
case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS:
return 0;
case KBASE_CONFIG_ATTR_PLATFORM_FUNCS:
return 0;
case KBASE_CONFIG_ATTR_CPU_SPEED_FUNC:
return DEFAULT_CPU_SPEED_FUNC;
case KBASE_CONFIG_ATTR_GPU_SPEED_FUNC:
return 0;
case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_DVFS_FREQ:
return DEFAULT_PM_DVFS_FREQ;
case KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS:
return DEFAULT_PM_GPU_POWEROFF_TICK_NS;
case KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_SHADER:
return DEFAULT_PM_POWEROFF_TICK_SHADER;
case KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_GPU:
return DEFAULT_PM_POWEROFF_TICK_GPU;
case KBASE_CONFIG_ATTR_POWER_MODEL_CALLBACKS:
return 0;
case KBASE_CONFIG_ATTR_SECURE_CALLBACKS:
return 0;
default:
dev_err(kbdev->dev, "kbasep_get_config_value. Cannot get value of attribute with id=%d and no default value defined", attribute_id);
return 0;
}
}
static void page_fault_worker(struct work_struct *data)
{
u64 fault_pfn;
u32 new_pages;
u32 fault_rel_pfn;
kbase_as * faulting_as;
int as_no;
kbase_context * kctx;
kbase_device * kbdev;
kbase_va_region *region;
mali_error err;
u32 fault_status;
faulting_as = container_of(data, kbase_as, work_pagefault);
fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
as_no = faulting_as->number;
kbdev = container_of( faulting_as, kbase_device, as[as_no] );
/* Grab the context that was already refcounted in kbase_mmu_interrupt().
* Therefore, it cannot be scheduled out of this AS until we explicitly release it
*
* NOTE: NULL can be returned here if we're gracefully handling a spurious interrupt */
kctx = kbasep_js_runpool_lookup_ctx_noretain( kbdev, as_no );
if ( kctx == NULL )
{
/* Address space has no context, terminate the work */
u32 reg;
/* AS transaction begin */
mutex_lock(&faulting_as->transaction_mutex);
reg = kbase_reg_read(kbdev, MMU_AS_REG(as_no, ASn_TRANSTAB_LO), NULL);
reg = (reg & (~(u32)MMU_TRANSTAB_ADRMODE_MASK)) | ASn_TRANSTAB_ADRMODE_UNMAPPED;
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_TRANSTAB_LO), reg, NULL);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_COMMAND), ASn_COMMAND_UPDATE, NULL);
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), (1UL << as_no), NULL);
mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
mmu_mask_reenable(kbdev, NULL, faulting_as);
return;
}
fault_status = kbase_reg_read(kbdev, MMU_AS_REG(as_no, ASn_FAULTSTATUS), NULL);
OSK_ASSERT( kctx->kbdev == kbdev );
kbase_gpu_vm_lock(kctx);
/* find the region object for this VA */
region = kbase_region_tracker_find_region_enclosing_address(kctx, faulting_as->fault_addr);
if (NULL == region || (GROWABLE_FLAGS_REQUIRED != (region->flags & GROWABLE_FLAGS_MASK)))
{
kbase_gpu_vm_unlock(kctx);
/* failed to find the region or mismatch of the flags */
kbase_mmu_report_fault_and_kill(kctx, faulting_as, faulting_as->fault_addr);
goto fault_done;
}
if ((((fault_status & ASn_FAULTSTATUS_ACCESS_TYPE_MASK) == ASn_FAULTSTATUS_ACCESS_TYPE_READ) &&
!(region->flags & KBASE_REG_GPU_RD)) ||
(((fault_status & ASn_FAULTSTATUS_ACCESS_TYPE_MASK) == ASn_FAULTSTATUS_ACCESS_TYPE_WRITE) &&
!(region->flags & KBASE_REG_GPU_WR)) ||
(((fault_status & ASn_FAULTSTATUS_ACCESS_TYPE_MASK) == ASn_FAULTSTATUS_ACCESS_TYPE_EX) &&
(region->flags & KBASE_REG_GPU_NX)))
{
OSK_PRINT_WARN(OSK_BASE_MMU, "Access permissions don't match: region->flags=0x%x", region->flags);
kbase_gpu_vm_unlock(kctx);
kbase_mmu_report_fault_and_kill(kctx, faulting_as, faulting_as->fault_addr);
goto fault_done;
}
/* find the size we need to grow it by */
/* we know the result fit in a u32 due to kbase_region_tracker_find_region_enclosing_address
* validating the fault_adress to be within a u32 from the start_pfn */
fault_rel_pfn = fault_pfn - region->start_pfn;
if (fault_rel_pfn < region->nr_alloc_pages)
{
OSK_PRINT_WARN(OSK_BASE_MMU, "Fault in allocated region of growable TMEM: Ignoring");
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), (1UL << as_no), NULL);
mmu_mask_reenable(kbdev, kctx, faulting_as);
kbase_gpu_vm_unlock(kctx);
goto fault_done;
}
new_pages = make_multiple(fault_rel_pfn - region->nr_alloc_pages + 1, region->extent);
if (new_pages + region->nr_alloc_pages > region->nr_pages)
{
/* cap to max vsize */
new_pages = region->nr_pages - region->nr_alloc_pages;
}
if (0 == new_pages)
{
/* Duplicate of a fault we've already handled, nothing to do */
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), (1UL << as_no), NULL);
mmu_mask_reenable(kbdev, kctx, faulting_as);
kbase_gpu_vm_unlock(kctx);
goto fault_done;
}
if (MALI_ERROR_NONE == kbase_alloc_phy_pages_helper(region, new_pages))
{
/* alloc success */
mali_addr64 lock_addr;
OSK_ASSERT(region->nr_alloc_pages <= region->nr_pages);
/* AS transaction begin */
mutex_lock(&faulting_as->transaction_mutex);
/* Lock the VA region we're about to update */
lock_addr = lock_region(kbdev, faulting_as->fault_addr >> PAGE_SHIFT, new_pages);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_LOCKADDR_LO), lock_addr & 0xFFFFFFFFUL, kctx);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_LOCKADDR_HI), lock_addr >> 32, kctx);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_COMMAND), ASn_COMMAND_LOCK, kctx);
/* set up the new pages */
err = kbase_mmu_insert_pages(kctx, region->start_pfn + region->nr_alloc_pages - new_pages,
®ion->phy_pages[region->nr_alloc_pages - new_pages],
new_pages, region->flags);
if(MALI_ERROR_NONE != err)
{
/* failed to insert pages, handle as a normal PF */
mutex_unlock(&faulting_as->transaction_mutex);
kbase_gpu_vm_unlock(kctx);
/* The locked VA region will be unlocked and the cache invalidated in here */
kbase_mmu_report_fault_and_kill(kctx, faulting_as, faulting_as->fault_addr);
goto fault_done;
}
#ifdef CONFIG_MALI_GATOR_SUPPORT
kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
#endif /* CONFIG_MALI_GATOR_SUPPORT */
/* clear the irq */
/* MUST BE BEFORE THE FLUSH/UNLOCK */
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), (1UL << as_no), NULL);
/* flush L2 and unlock the VA (resumes the MMU) */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
{
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_COMMAND), ASn_COMMAND_FLUSH, kctx);
}
else
{
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_COMMAND), ASn_COMMAND_FLUSH_PT, kctx);
}
/* wait for the flush to complete */
while (kbase_reg_read(kbdev, MMU_AS_REG(as_no, ASn_STATUS), kctx) & 1);
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630))
{
/* Issue an UNLOCK command to ensure that valid page tables are re-read by the GPU after an update.
Note that, the FLUSH command should perform all the actions necessary, however the bus logs show
that if multiple page faults occur within an 8 page region the MMU does not always re-read the
updated page table entries for later faults or is only partially read, it subsequently raises the
page fault IRQ for the same addresses, the unlock ensures that the MMU cache is flushed, so updates
can be re-read. As the region is now unlocked we need to issue 2 UNLOCK commands in order to flush the
MMU/uTLB, see PRLAM-8812.
*/
kbase_reg_write(kctx->kbdev, MMU_AS_REG(kctx->as_nr, ASn_COMMAND), ASn_COMMAND_UNLOCK, kctx);
kbase_reg_write(kctx->kbdev, MMU_AS_REG(kctx->as_nr, ASn_COMMAND), ASn_COMMAND_UNLOCK, kctx);
}
mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
/* reenable this in the mask */
mmu_mask_reenable(kbdev, kctx, faulting_as);
kbase_gpu_vm_unlock(kctx);
}
/**
* @brief Process a replay job
*
* Called from kbase_process_soft_job.
*
* On exit, if the job has completed, katom->event_code will have been updated.
* If the job has not completed, and is replaying jobs, then the atom status
* will have been reset to KBASE_JD_ATOM_STATE_QUEUED.
*
* @param[in] katom The atom to be processed
* @return false if the atom has completed
* true if the atom is replaying jobs
*/
bool kbase_replay_process(struct kbase_jd_atom *katom)
{
struct kbase_context *kctx = katom->kctx;
struct kbase_jd_context *jctx = &kctx->jctx;
struct kbase_device *kbdev = kctx->kbdev;
/* Don't replay this atom if these issues are not present in the
* hardware */
if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11020) &&
!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11024)) {
dev_dbg(kbdev->dev, "Hardware does not need replay workaround");
/* Signal failure to userspace */
katom->event_code = BASE_JD_EVENT_JOB_INVALID;
return false;
}
if (katom->event_code == BASE_JD_EVENT_DONE) {
dev_dbg(kbdev->dev, "Previous job succeeded - not replaying\n");
if (katom->retry_count)
kbase_disjoint_state_down(kbdev);
return false;
}
if (jctx->sched_info.ctx.is_dying) {
dev_dbg(kbdev->dev, "Not replaying; context is dying\n");
if (katom->retry_count)
kbase_disjoint_state_down(kbdev);
return false;
}
/* Check job exception type and source before replaying. */
if (!kbase_replay_fault_check(katom)) {
dev_dbg(kbdev->dev,
"Replay cancelled on event %x\n", katom->event_code);
/* katom->event_code is already set to the failure code of the
* previous job.
*/
return false;
}
dev_warn(kbdev->dev, "Replaying jobs retry=%d\n",
katom->retry_count);
katom->retry_count++;
if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) {
dev_err(kbdev->dev, "Replay exceeded limit - failing jobs\n");
kbase_disjoint_state_down(kbdev);
/* katom->event_code is already set to the failure code of the
previous job */
return false;
}
/* only enter the disjoint state once for the whole time while the replay is ongoing */
if (katom->retry_count == 1)
kbase_disjoint_state_up(kbdev);
INIT_WORK(&katom->work, kbase_replay_process_worker);
queue_work(kctx->event_workq, &katom->work);
return true;
}
