void kbasep_js_remove_job( kbase_context *kctx, kbase_jd_atom *atom )
{
kbasep_js_policy_cfs_ctx *ctx_info;
kbasep_js_kctx_info *js_kctx_info;
kbase_device *kbdev;
kbasep_js_device_data *js_devdata;
kbasep_js_policy *js_policy;
mali_bool nss_state_changed;
OSK_ASSERT( kctx != NULL );
OSK_ASSERT( atom != NULL );
kbdev = kctx->kbdev;
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
/* De-refcount ctx.nr_jobs */
OSK_ASSERT( js_kctx_info->ctx.nr_jobs > 0 );
--(js_kctx_info->ctx.nr_jobs);
ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
/* Adjust context priority to no longer include removed job */
OSK_ASSERT(ctx_info->bag_total_nr_atoms > 0);
ctx_info->bag_total_nr_atoms--;
ctx_info->bag_total_priority -= atom->nice_prio;
OSK_ASSERT(ctx_info->bag_total_priority >= 0);
/* Get average priority and convert to NICE range -20..19 */
if(ctx_info->bag_total_nr_atoms)
{
ctx_info->bag_priority = (ctx_info->bag_total_priority / ctx_info->bag_total_nr_atoms) - 20;
}
osk_spinlock_irq_lock( &js_devdata->runpool_irq.lock );
nss_state_changed = kbasep_js_check_and_deref_nss_job( js_devdata, kctx, atom );
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
/* De-register the job from the system */
kbasep_js_policy_term_job( js_policy, atom );
/* A change in NSS state means we might be able to run on slots that were
* previously empty, but could now run jobs on them */
if ( nss_state_changed != MALI_FALSE )
{
osk_mutex_lock( &js_devdata->runpool_mutex );
kbasep_js_try_run_next_job( kbdev );
osk_mutex_unlock( &js_devdata->runpool_mutex );
}
}
/*
* Functions private to KBase ('Protected' functions)
*/
void kbase_js_try_run_jobs( kbase_device *kbdev )
{
kbasep_js_device_data *js_devdata;
OSK_ASSERT( kbdev != NULL );
js_devdata = &kbdev->js_data;
osk_mutex_lock( &js_devdata->runpool_mutex );
kbasep_js_try_run_next_job( kbdev );
osk_mutex_unlock( &js_devdata->runpool_mutex );
}
/**
* Picks a free address space and add the context to the Policy. Then perform a
* transaction on this AS and RunPool IRQ to:
* - setup the runpool_irq structure and the context on that AS
* - Activate the MMU on that AS
* - Allow jobs to be submitted on that AS
*
* Locking conditions:
* - Caller must hold the kbasep_js_kctx_info::jsctx_mutex
* - Caller must hold the kbase_js_device_data::runpool_mutex
* - AS transaction mutex will be obtained
* - Runpool IRQ lock will be obtained
*/
STATIC void assign_and_activate_kctx_addr_space( kbase_device *kbdev, kbase_context *kctx )
{
kbasep_js_device_data *js_devdata;
kbase_as *current_as;
kbasep_js_per_as_data *js_per_as_data;
long ffs_result;
OSK_ASSERT( kbdev != NULL );
OSK_ASSERT( kctx != NULL );
js_devdata = &kbdev->js_data;
/* Find the free address space */
ffs_result = osk_find_first_set_bit( js_devdata->as_free );
/* ASSERT that we should've found a free one */
OSK_ASSERT( 0 <= ffs_result && ffs_result < kbdev->nr_address_spaces );
js_devdata->as_free &= ~((u16)(1u << ffs_result));
/*
* Transaction on the AS and runpool_irq
*/
current_as = &kbdev->as[ffs_result];
js_per_as_data = &js_devdata->runpool_irq.per_as_data[ffs_result];
osk_mutex_lock( ¤t_as->transaction_mutex );
osk_spinlock_irq_lock( &js_devdata->runpool_irq.lock );
/* NSS Handling */
kbasep_js_check_and_ref_nss_running_ctx( js_devdata, kctx );
/* Assign addr space */
kctx->as_nr = (int)ffs_result;
/* Activate this address space on the MMU */
kbase_mmu_update( kctx );
/* Allow it to run jobs */
kbasep_js_set_submit_allowed( js_devdata, kctx );
/* Book-keeping */
js_per_as_data->kctx = kctx;
js_per_as_data->as_busy_refcount = 0;
/* Lastly, add the context to the policy's runpool - this really allows it to run jobs */
kbasep_js_policy_runpool_add_ctx( &js_devdata->policy, kctx );
/*
* Transaction complete
*/
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
osk_mutex_unlock( ¤t_as->transaction_mutex );
}
void kbasep_js_runpool_release_ctx( kbase_device *kbdev, kbase_context *kctx )
{
kbasep_js_device_data *js_devdata;
kbasep_js_kctx_info *js_kctx_info;
kbasep_js_policy *js_policy;
kbasep_js_per_as_data *js_per_as_data;
mali_bool was_descheduled = MALI_FALSE;
int saved_as_nr;
kbase_as *current_as;
int new_ref_count;
mali_bool nss_state_changed = MALI_FALSE;
OSK_ASSERT( kbdev != NULL );
OSK_ASSERT( kctx != NULL );
js_kctx_info = &kctx->jctx.sched_info;
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
osk_mutex_lock( &js_kctx_info->ctx.jsctx_mutex );
osk_mutex_lock( &js_devdata->runpool_mutex );
/* Ensure context really is scheduled in */
OSK_ASSERT( js_kctx_info->ctx.is_scheduled != MALI_FALSE );
/* The saved_as_nr must be accessed under lock, but we also need to take a
* sleeping mutex. Since the ctx is known to be busy-refcounted, we can
* just take the runpool lock briefly, then taken it again later (the as_nr
* won't be reassigned due to being busy).
*
* We ASSERT on this fact */
osk_spinlock_irq_lock( &js_devdata->runpool_irq.lock );
{
saved_as_nr = kctx->as_nr;
OSK_ASSERT( saved_as_nr != KBASEP_AS_NR_INVALID );
js_per_as_data = &js_devdata->runpool_irq.per_as_data[saved_as_nr];
OSK_ASSERT( js_per_as_data->as_busy_refcount > 0 );
}
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
/*
* Transaction begins on AS and runpool_irq
*
* Doubly-assert that our previous facts are still true
*/
current_as = &kbdev->as[saved_as_nr];
osk_mutex_lock( ¤t_as->transaction_mutex );
osk_spinlock_irq_lock( &js_devdata->runpool_irq.lock );
OSK_ASSERT( saved_as_nr == kctx->as_nr );
OSK_ASSERT( js_per_as_data->as_busy_refcount > 0 );
/* Update refcount */
new_ref_count = --(js_per_as_data->as_busy_refcount);
/* Make a set of checks to see if the context should be scheduled out */
if ( new_ref_count == 0
&& ( kctx->jctx.sched_info.ctx.nr_jobs == 0
|| kbasep_js_is_submit_allowed( js_devdata, kctx ) == MALI_FALSE ) )
{
/* Last reference, and we've been told to remove this context from the Run Pool */
OSK_PRINT_INFO(OSK_BASE_JM, "JS: RunPool Remove Context %p because as_busy_refcount=%d, jobs=%d, allowed=%d",
kctx,
new_ref_count,
js_kctx_info->ctx.nr_jobs,
kbasep_js_is_submit_allowed( js_devdata, kctx ) );
kbasep_js_policy_runpool_remove_ctx( js_policy, kctx );
/* Stop any more refcounts occuring on the context */
js_per_as_data->kctx = NULL;
/* Ensure we prevent the context from submitting any new jobs
* e.g. from kbasep_js_try_run_next_job_on_slot_irq_nolock() */
kbasep_js_clear_submit_allowed( js_devdata, kctx );
/* Disable the MMU on the affected address space, and indicate it's invalid */
kbase_mmu_disable( kctx );
kctx->as_nr = KBASEP_AS_NR_INVALID;
/* NSS handling */
nss_state_changed = kbasep_js_check_and_deref_nss_running_ctx( js_devdata, kctx );
/*
* Transaction ends on AS and runpool_irq:
*
* By this point, the AS-related data is now clear and ready for re-use.
*
* Since releases only occur once for each previous successful retain, and no more
* retains are allowed on this context, no other thread will be operating in this
* code whilst we are
*/
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
osk_mutex_unlock( ¤t_as->transaction_mutex );
/* Free up the address space */
js_devdata->as_free |= ((u16)(1u << saved_as_nr));
/* Note: Don't reuse saved_as_nr now */
/* update book-keeping info */
--(js_devdata->nr_contexts_running);
js_kctx_info->ctx.is_scheduled = MALI_FALSE;
/* Signal any waiter that the context is not scheduled, so is safe for
* termination - once the jsctx_mutex is also dropped, and jobs have
* finished. */
osk_waitq_set( &js_kctx_info->ctx.not_scheduled_waitq );
/* Handle dying contexts */
if ( js_kctx_info->ctx.is_dying != MALI_FALSE )
{
/* This happens asynchronously */
OSK_PRINT_INFO(OSK_BASE_JM, "JS: ** Killing Context %p on RunPool Remove **", kctx );
kbasep_js_policy_kill_all_ctx_jobs( js_policy, kctx );
}
/* Queue an action to occur after we've dropped the lock */
was_descheduled = MALI_TRUE;
}
else
{
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
osk_mutex_unlock( ¤t_as->transaction_mutex );
}
osk_mutex_unlock( &js_devdata->runpool_mutex );
/* Do we have an action queued whilst the lock was held? */
if ( was_descheduled != MALI_FALSE )
{
/* Determine whether this context should be requeued on the policy queue */
if ( js_kctx_info->ctx.nr_jobs > 0 && js_kctx_info->ctx.is_dying == MALI_FALSE )
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Requeue Context %p", kctx );
osk_mutex_lock( &js_devdata->queue_mutex );
kbasep_js_policy_enqueue_ctx( js_policy, kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
}
else
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Idling Context %p (not requeued)", kctx );
/* Notify PM that a context has gone idle */
kbase_pm_context_idle(kctx->kbdev);
}
}
/* We've finished with this context for now, so drop the lock for it. */
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
if ( was_descheduled != MALI_FALSE )
{
/* We've freed up an address space, so let's try to schedule in another
* context
*
* Note: if there's a context to schedule in, then it also tries to run
* another job, in case the new context has jobs satisfying requirements
* that no other context/job in the runpool does */
kbasep_js_try_schedule_head_ctx( kbdev );
}
if ( nss_state_changed != MALI_FALSE )
{
osk_mutex_lock( &js_devdata->runpool_mutex );
kbasep_js_try_run_next_job( kbdev );
osk_mutex_unlock( &js_devdata->runpool_mutex );
}
}
mali_bool kbasep_js_add_job( kbase_context *kctx, kbase_jd_atom *atom )
{
kbasep_js_kctx_info *js_kctx_info;
kbase_device *kbdev;
kbasep_js_device_data *js_devdata;
kbasep_js_policy *js_policy;
mali_bool policy_queue_updated = MALI_FALSE;
OSK_ASSERT( kctx != NULL );
OSK_ASSERT( atom != NULL );
kbdev = kctx->kbdev;
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
osk_mutex_lock( &js_devdata->runpool_mutex );
OSK_PRINT_INFO( OSK_BASE_JM, "JS: job enqueue %p", (void *)atom);
/* Refcount ctx.nr_jobs */
OSK_ASSERT( js_kctx_info->ctx.nr_jobs < U32_MAX );
++(js_kctx_info->ctx.nr_jobs);
/* Setup any scheduling information */
kbasep_js_clear_job_retry_submit( atom );
/*
* Begin Runpool_irq transaction
*/
osk_spinlock_irq_lock( &js_devdata->runpool_irq.lock );
{
/* NSS refcounting */
kbasep_js_check_and_ref_nss_job( js_devdata, kctx, atom );
/* Enqueue the job in the policy, causing it to be scheduled if the
* parent context gets scheduled */
kbasep_js_policy_enqueue_job( js_policy, atom );
}
osk_spinlock_irq_unlock( &js_devdata->runpool_irq.lock );
/* End runpool_irq transaction */
if ( js_kctx_info->ctx.is_scheduled != MALI_FALSE )
{
/* Handle an already running context - try to run the new job, in case it
* matches requirements that aren't matched by any other job in the Run
* Pool */
kbasep_js_try_run_next_job( kbdev );
}
osk_mutex_unlock( &js_devdata->runpool_mutex );
if ( js_kctx_info->ctx.is_scheduled == MALI_FALSE && js_kctx_info->ctx.nr_jobs == 1 )
{
/* Handle Refcount going from 0 to 1: schedule the context on the Policy Queue */
OSK_ASSERT( js_kctx_info->ctx.is_scheduled == MALI_FALSE );
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Enqueue Context %p", kctx );
osk_mutex_lock( &js_devdata->queue_mutex );
kbasep_js_policy_enqueue_ctx( js_policy, kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
/* If the runpool is full and this job has a higher priority than the non-running
* job in the runpool - evict it so this higher priority job starts faster */
kbasep_js_runpool_attempt_fast_start_ctx( kbdev, kctx );
/* This context is becoming active */
kbase_pm_context_active(kctx->kbdev);
/* NOTE: Potentially, we can make the scheduling of the head context
* happen in a work-queue if we need to wait for the PM to power
* up. Also need logic to submit nothing until PM really has completed
* powering up. */
/* Policy Queue was updated - caller must try to schedule the head context */
policy_queue_updated = MALI_TRUE;
}
return policy_queue_updated;
}
/**
* Fast start a higher priority job when the runpool is full and contains a
* non-running lower priority job.
* The evicted job will be returned to the policy queue.
*
* The following locking conditions are made on the caller:
* - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
*/
STATIC void kbasep_js_runpool_attempt_fast_start_ctx( kbase_device *kbdev, kbase_context *kctx )
{
kbasep_js_device_data *js_devdata;
kbasep_js_policy *js_policy;
kbasep_js_per_as_data *js_per_as_data;
int evict_as_nr;
kbase_as *current_as;
mali_bool nss_state_changed = MALI_FALSE;
mali_bool is_runpool_full;
OSK_ASSERT(kbdev != NULL);
OSK_ASSERT(kctx != NULL);
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
osk_mutex_lock(&js_devdata->runpool_mutex);
/* If the runpool is full, attempt to fast start our context */
is_runpool_full = (mali_bool)(js_devdata->nr_contexts_running >= kbdev->nr_address_spaces);
if(is_runpool_full != MALI_FALSE)
{
/* No free address spaces - attempt to evict non-running lower priority context */
osk_spinlock_irq_lock(&js_devdata->runpool_irq.lock);
for(evict_as_nr = 0; evict_as_nr < kbdev->nr_address_spaces; evict_as_nr++)
{
current_as = &kbdev->as[evict_as_nr];
js_per_as_data = &js_devdata->runpool_irq.per_as_data[evict_as_nr];
/* Look for the AS which is not currently running */
if(0 == js_per_as_data->as_busy_refcount)
{
kbase_context *kctx_evict = js_per_as_data->kctx;
osk_spinlock_irq_unlock(&js_devdata->runpool_irq.lock);
/* Now compare the scheduled priority we are considering evicting with the new ctx priority
* and take into consideration if the scheduled priority is a realtime policy or not.
* Note that the lower the number, the higher the priority
*/
if(((kctx_evict->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy == MALI_FALSE) &&
kctx->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy) ||
((kctx_evict->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy ==
kctx->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy) &&
(kctx_evict->jctx.sched_info.runpool.policy_ctx.cfs.bag_priority >
kctx->jctx.sched_info.runpool.policy_ctx.cfs.bag_priority)))
{
/* Evict idle job in the runpool as priority is lower than new job */
osk_mutex_lock(¤t_as->transaction_mutex);
osk_spinlock_irq_lock(&js_devdata->runpool_irq.lock);
/* Remove the context from the runpool policy list (policy_info->scheduled_ctxs_head) */
kbasep_js_policy_runpool_remove_ctx(js_policy, kctx_evict);
/* Stop any more refcounts occuring on the context */
js_per_as_data->kctx = NULL;
/* Prevent a context from submitting more jobs on this policy */
kbasep_js_clear_submit_allowed(js_devdata, kctx_evict);
/* Disable the MMU on the affected address space, and indicate it's invalid */
kbase_mmu_disable(kctx_evict);
kctx_evict->as_nr = KBASEP_AS_NR_INVALID;
/* NSS handling */
nss_state_changed = kbasep_js_check_and_deref_nss_running_ctx(js_devdata, kctx_evict);
CSTD_UNUSED(nss_state_changed);
osk_spinlock_irq_unlock(&js_devdata->runpool_irq.lock);
osk_mutex_unlock(¤t_as->transaction_mutex);
/* Free up the address space */
js_devdata->as_free |= ((u16)(1u << evict_as_nr));
/* update book-keeping info */
--(js_devdata->nr_contexts_running);
kctx_evict->jctx.sched_info.ctx.is_scheduled = MALI_FALSE;
/* Signal any waiter that the context is not scheduled */
osk_waitq_set(&kctx_evict->jctx.sched_info.ctx.not_scheduled_waitq);
osk_mutex_unlock(&js_devdata->runpool_mutex);
/* Requeue onto the policy queue */
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Requeue Context %p", kctx_evict);
osk_mutex_lock(&js_devdata->queue_mutex);
kbasep_js_policy_enqueue_ctx(js_policy, kctx_evict);
osk_mutex_unlock(&js_devdata->queue_mutex);
/* ctx fast start has taken place */
return;
}
osk_spinlock_irq_lock(&js_devdata->runpool_irq.lock);
}
}
osk_spinlock_irq_unlock(&js_devdata->runpool_irq.lock);
}
/* ctx fast start has not taken place */
osk_mutex_unlock(&js_devdata->runpool_mutex);
}
void kbasep_js_try_schedule_head_ctx( kbase_device *kbdev )
{
kbasep_js_device_data *js_devdata;
mali_bool has_kctx;
kbase_context *head_kctx;
kbasep_js_kctx_info *js_kctx_info;
mali_bool is_runpool_full;
OSK_ASSERT( kbdev != NULL );
js_devdata = &kbdev->js_data;
/* Make a speculative check on the Run Pool - this MUST be repeated once
* we've obtained a context from the queue and reobtained the Run Pool
* lock */
osk_mutex_lock( &js_devdata->runpool_mutex );
is_runpool_full = (mali_bool)( js_devdata->nr_contexts_running >= kbdev->nr_address_spaces );
osk_mutex_unlock( &js_devdata->runpool_mutex );
if ( is_runpool_full != MALI_FALSE )
{
/* No free address spaces - nothing to do */
return;
}
/* Grab the context off head of queue - if there is one */
osk_mutex_lock( &js_devdata->queue_mutex );
has_kctx = kbasep_js_policy_dequeue_head_ctx( &js_devdata->policy, &head_kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
if ( has_kctx == MALI_FALSE )
{
/* No ctxs to run - nothing to do */
return;
}
js_kctx_info = &head_kctx->jctx.sched_info;
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Dequeue Context %p", head_kctx );
/*
* Atomic transaction on the Context and Run Pool begins
*/
osk_mutex_lock( &js_kctx_info->ctx.jsctx_mutex );
osk_mutex_lock( &js_devdata->runpool_mutex );
/* Re-check to see if the Run Pool is full */
is_runpool_full = (mali_bool)( js_devdata->nr_contexts_running >= kbdev->nr_address_spaces );
if ( is_runpool_full != MALI_FALSE )
{
/* No free address spaces - roll back the transaction so far and return */
osk_mutex_unlock( &js_devdata->runpool_mutex );
/* Only requeue if not dying - which might occur through zapping-whilst-scheduling */
if ( js_kctx_info->ctx.is_dying == MALI_FALSE )
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Transaction rollback: Requeue Context %p", head_kctx );
osk_mutex_lock( &js_devdata->queue_mutex );
kbasep_js_policy_enqueue_ctx( &js_devdata->policy, head_kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
}
else
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Transaction rollback: Context %p is dying. Kill remaining jobs and pm-idle ctx", head_kctx );
OSK_ASSERT( js_kctx_info->ctx.nr_jobs > 0 );
/* Notify PM that a context has gone idle */
kbase_pm_context_idle(kbdev);
/* Kill all the jobs present (call kbase_jd_cancel on all jobs) */
kbasep_js_policy_kill_all_ctx_jobs( &js_devdata->policy, head_kctx );
/* Nothing more to be done to kill the context here, kbase_jd_zap_context
* waits for all jobs to be cancelled */
}
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
return;
}
OSK_PRINT_INFO(OSK_BASE_JM, "JS: RunPool Add Context %p", head_kctx );
/* update book-keeping info */
js_kctx_info->ctx.is_scheduled = MALI_TRUE;
++(js_devdata->nr_contexts_running);
/* Cause any future waiter-on-termination to wait until the context is
* descheduled */
osk_waitq_clear( &js_kctx_info->ctx.not_scheduled_waitq );
/* Do all the necessaries to pick the address space (inc. update book-keeping info)
* Add the context to the Run Pool, and allow it to run jobs */
assign_and_activate_kctx_addr_space( kbdev, head_kctx );
/* Check and setup HW counters dumping */
osk_spinlock_lock(&kbdev->hwcnt_lock);
osk_spinlock_irq_lock(&js_devdata->runpool_irq.lock);
if (head_kctx == kbdev->hwcnt_context &&
kbdev->hwcnt_is_setup == MALI_FALSE)
{
/* Setup the base address */
#if BASE_HW_ISSUE_8186
u32 val;
/* Save and clear PRFCNT_TILER_EN */
val = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), head_kctx);
if(0 != val)
{
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, head_kctx);
}
/* Update PRFCNT_CONFIG with TILER_EN = 0 */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (head_kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, head_kctx);
/* Restore PRFCNT_TILER_EN */
if(0 != val)
{
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), val, head_kctx);
}
#else
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (head_kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, head_kctx);
#endif
/* Prevent the context to be scheduled out */
kbasep_js_runpool_retain_ctx_nolock(kbdev, head_kctx);
kbdev->hwcnt_is_setup = MALI_TRUE;
}
osk_spinlock_irq_unlock(&js_devdata->runpool_irq.lock);
osk_spinlock_unlock(&kbdev->hwcnt_lock);
/* Try to run the next job, in case this context has jobs that match the
* job slot requirements, but none of the other currently running contexts
* do */
kbasep_js_try_run_next_job( kbdev );
/* Transaction complete */
osk_mutex_unlock( &js_devdata->runpool_mutex );
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
/* Note: after this point, the context could potentially get scheduled out immediately */
}
static void page_fault_worker(osk_workq_work *data)
{
u64 fault_pfn;
u32 new_pages;
u32 fault_rel_pfn;
kbase_as * faulting_as;
int as_no;
kbase_context * kctx;
kbase_device * kbdev;
kbase_va_region *region;
mali_error err;
faulting_as = CONTAINER_OF(data, kbase_as, work_pagefault);
fault_pfn = faulting_as->fault_addr >> OSK_PAGE_SHIFT;
as_no = faulting_as->number;
kbdev = CONTAINER_OF( faulting_as, kbase_device, as[as_no] );
/* Grab the context that was already refcounted in kbase_mmu_interrupt().
* Therefore, it cannot be scheduled out of this AS until we explicitly release it
*
* NOTE: NULL can be returned here if we're gracefully handling a spurious interrupt */
kctx = kbasep_js_runpool_lookup_ctx_noretain( kbdev, as_no );
if ( kctx == NULL )
{
/* Address space has no context, terminate the work */
u32 reg;
/* AS transaction begin */
osk_mutex_lock(&faulting_as->transaction_mutex);
reg = kbase_reg_read(kbdev, MMU_AS_REG(as_no, ASn_TRANSTAB_LO), NULL);
reg = (reg & (~(u32)MMU_TRANSTAB_ADRMODE_MASK)) | ASn_TRANSTAB_ADRMODE_UNMAPPED;
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_TRANSTAB_LO), reg, NULL);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_COMMAND), ASn_COMMAND_UPDATE, NULL);
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), (1UL << as_no), NULL);
osk_mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
mmu_mask_reenable(kbdev, NULL, faulting_as);
return;
}
OSK_ASSERT( kctx->kbdev == kbdev );
kbase_gpu_vm_lock(kctx);
/* find the region object for this VA */
region = kbase_region_lookup(kctx, faulting_as->fault_addr);
if (NULL == region || (GROWABLE_FLAGS_REQUIRED != (region->flags & GROWABLE_FLAGS_MASK)))
{
kbase_gpu_vm_unlock(kctx);
/* failed to find the region or mismatch of the flags */
kbase_mmu_report_fault_and_kill(kctx, faulting_as, faulting_as->fault_addr);
goto fault_done;
}
/* find the size we need to grow it by */
/* we know the result fit in a u32 due to kbase_region_lookup
* validating the fault_adress to be within a u32 from the start_pfn */
fault_rel_pfn = fault_pfn - region->start_pfn;
new_pages = make_multiple(fault_rel_pfn - region->nr_alloc_pages + 1, region->extent);
if (new_pages + region->nr_alloc_pages > region->nr_pages)
{
/* cap to max vsize */
new_pages = region->nr_pages - region->nr_alloc_pages;
}
if (0 == new_pages)
{
/* Duplicate of a fault we've already handled, nothing to do */
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), (1UL << as_no), NULL);
mmu_mask_reenable(kbdev, kctx, faulting_as);
kbase_gpu_vm_unlock(kctx);
goto fault_done;
}
if (MALI_ERROR_NONE == kbase_alloc_phy_pages_helper(region, new_pages))
{
/* alloc success */
mali_addr64 lock_addr;
OSK_ASSERT(region->nr_alloc_pages <= region->nr_pages);
/* AS transaction begin */
osk_mutex_lock(&faulting_as->transaction_mutex);
/* Lock the VA region we're about to update */
lock_addr = lock_region(faulting_as->fault_addr >> OSK_PAGE_SHIFT, new_pages);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_LOCKADDR_LO), lock_addr & 0xFFFFFFFFUL, kctx);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_LOCKADDR_HI), lock_addr >> 32, kctx);
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_COMMAND), ASn_COMMAND_LOCK, kctx);
/* set up the new pages */
err = kbase_mmu_insert_pages(kctx, region->start_pfn + region->nr_alloc_pages - new_pages,
®ion->phy_pages[region->nr_alloc_pages - new_pages],
new_pages, region->flags);
if(MALI_ERROR_NONE != err)
{
/* failed to insert pages, handle as a normal PF */
osk_mutex_unlock(&faulting_as->transaction_mutex);
kbase_gpu_vm_unlock(kctx);
/* The locked VA region will be unlocked and the cache invalidated in here */
kbase_mmu_report_fault_and_kill(kctx, faulting_as, faulting_as->fault_addr);
goto fault_done;
}
/* clear the irq */
/* MUST BE BEFORE THE FLUSH/UNLOCK */
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), (1UL << as_no), NULL);
/* flush L2 and unlock the VA (resumes the MMU) */
kbase_reg_write(kbdev, MMU_AS_REG(as_no, ASn_COMMAND), ASn_COMMAND_FLUSH, kctx);
/* wait for the flush to complete */
while (kbase_reg_read(kbdev, MMU_AS_REG(as_no, ASn_STATUS), kctx) & 1);
osk_mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
/* reenable this in the mask */
mmu_mask_reenable(kbdev, kctx, faulting_as);
kbase_gpu_vm_unlock(kctx);
}
