/**
* @brief Issue Cache Clean & Invalidate command to hardware
*/
static void kbasep_instr_hwcnt_cacheclean(kbase_device *kbdev)
{
unsigned long flags;
unsigned long pm_flags;
u32 irq_mask;
KBASE_DEBUG_ASSERT(NULL != kbdev);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
/* Wait for any reset to complete */
while (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.cache_clean_wait,
kbdev->hwcnt.state != KBASE_INSTR_STATE_RESETTING);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_REQUEST_CLEAN);
/* Enable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | CLEAN_CACHES_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES, NULL);
kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANING;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
}
/**
* @brief Cache clean interrupt received
*/
void kbase_clean_caches_done(kbase_device *kbdev)
{
u32 irq_mask;
if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
unsigned long flags;
unsigned long pm_flags;
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
/* Disable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~CLEAN_CACHES_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* Wakeup... */
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_CLEANING) {
/* Only wake if we weren't resetting */
kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANED;
wake_up(&kbdev->hwcnt.cache_clean_wait);
}
/* NOTE: In the state KBASE_INSTR_STATE_RESETTING, We're in a reset,
* and the instrumentation state hasn't been restored yet -
* kbasep_reset_timeout_worker() will do the rest of the work */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
}
}
/**
* @brief Clear the HW counters
*/
mali_error kbase_instr_hwcnt_clear(kbase_context *kctx)
{
unsigned long flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
kbase_device *kbdev;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
/* Check it's the context previously set up and we're not already dumping */
if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE)
goto out;
/* Clear the counters */
KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR, kctx);
err = MALI_ERROR_NONE;
out:
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
void kbase_wait_write_flush(struct kbase_context *kctx)
{
u32 base_count = 0;
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread */
kbase_pm_context_active(kctx->kbdev);
kbase_pm_request_gpu_cycle_counter(kctx->kbdev);
while (true) {
u32 new_count;
new_count = kbase_reg_read(kctx->kbdev,
GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
/* First time around, just store the count. */
if (base_count == 0) {
base_count = new_count;
continue;
}
/* No need to handle wrapping, unsigned maths works for this. */
if ((new_count - base_count) > 1000)
break;
}
kbase_pm_release_gpu_cycle_counter(kctx->kbdev);
kbase_pm_context_idle(kctx->kbdev);
}
/**
* @brief Issue Dump command to hardware
*
* Notes:
* - does not sleep
*/
mali_error kbase_instr_hwcnt_dump_irq(struct kbase_context *kctx)
{
unsigned long flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
struct kbase_device *kbdev;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.kctx != kctx) {
/* The instrumentation has been setup for another context */
GPU_LOG(DVFS_INFO, DUMMY, 0u, 0u, "hwcnt irq error in %s %d \n", __FUNCTION__, err);
goto unlock;
}
if (kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE) {
/* HW counters are disabled or another dump is ongoing, or we're resetting */
GPU_LOG(DVFS_INFO, DUMMY, 0u, 0u, "hwcnt disabled or another dump is ongoing in %s %d \n", __FUNCTION__, err);
goto unlock;
}
kbdev->hwcnt.triggered = 0;
/* Mark that we're dumping - the PF handler can signal that we faulted */
kbdev->hwcnt.state = KBASE_INSTR_STATE_DUMPING;
/* Reconfigure the dump address */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kbdev->hwcnt.addr & 0xFFFFFFFF, NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kbdev->hwcnt.addr >> 32, NULL);
/* Start dumping */
KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, kbdev->hwcnt.addr, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE, kctx);
dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx);
err = MALI_ERROR_NONE;
unlock:
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
static int gpu_set_maximum_outstanding_req(int val)
{
volatile unsigned int reg;
if (val > 0b1111)
return -1;
if (!pkbdev)
return -2;
if (!gpu_is_power_on())
return -3;
reg = kbase_os_reg_read(pkbdev, GPU_CONTROL_REG(L2_MMU_CONFIG));
reg &= ~(0b1111 << 24);
reg |= ((val & 0b1111) << 24);
kbase_os_reg_write(pkbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), reg);
return 0;
}
static void pm_callback_power_off(struct kbase_device *kbdev)
{
#if HARD_RESET_AT_POWER_OFF
/* Cause a GPU hard reset to test whether we have actually idled the GPU
* and that we properly reconfigure the GPU on power up.
* Usually this would be dangerous, but if the GPU is working correctly it should
* be completely safe as the GPU should not be active at this point.
* However this is disabled normally because it will most likely interfere with
* bus logging etc.
*/
KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
#endif
}
mali_error kbase_pm_powerup(kbase_device *kbdev)
{
unsigned long flags;
mali_error ret;
KBASE_DEBUG_ASSERT(kbdev != NULL);
mutex_lock(&kbdev->pm.lock);
/* A suspend won't happen during startup/insmod */
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
/* Power up the GPU, don't enable IRQs as we are not ready to receive them. */
ret = kbase_pm_init_hw(kbdev, MALI_FALSE );
if (ret != MALI_ERROR_NONE) {
mutex_unlock(&kbdev->pm.lock);
return ret;
}
kbasep_pm_read_present_cores(kbdev);
kbdev->pm.debug_core_mask = kbdev->shader_present_bitmap;
/* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
kbdev->pm.active_count = 1;
spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* Ensure cycle counter is off */
kbdev->pm.gpu_cycle_counter_requests = 0;
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* We are ready to receive IRQ's now as power policy is set up, so enable them now. */
#ifdef CONFIG_MALI_DEBUG
spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
kbdev->pm.driver_ready_for_irqs = MALI_TRUE;
spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
#endif
kbase_pm_enable_interrupts(kbdev);
/* Turn on the GPU and any cores needed by the policy */
kbase_pm_do_poweron(kbdev);
mutex_unlock(&kbdev->pm.lock);
/* Idle the GPU and/or cores, if the policy wants it to */
kbase_pm_context_idle(kbdev);
return MALI_ERROR_NONE;
}
mali_error kbase_pm_powerup(kbase_device *kbdev)
{
unsigned long flags;
mali_error ret;
OSK_ASSERT(kbdev != NULL);
ret = kbase_pm_init_hw(kbdev);
if (ret != MALI_ERROR_NONE)
{
return ret;
}
kbase_pm_power_transitioning(kbdev);
kbasep_pm_read_present_cores(kbdev);
/* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
spin_lock_irqsave(&kbdev->pm.active_count_lock, flags);
kbdev->pm.active_count = 1;
spin_unlock_irqrestore(&kbdev->pm.active_count_lock, flags);
spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
/* Ensure cycle counter is off */
kbdev->pm.gpu_cycle_counter_requests = 0;
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
atomic_set(&kbdev->pm.pending_events, 0);
atomic_set(&kbdev->pm.work_active, KBASE_PM_WORK_ACTIVE_STATE_INACTIVE);
kbdev->pm.new_policy = NULL;
kbdev->pm.current_policy = policy_list[0];
KBASE_TRACE_ADD( kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, kbdev->pm.current_policy->id );
kbdev->pm.current_policy->init(kbdev);
kbase_pm_send_event(kbdev, KBASE_PM_EVENT_POLICY_INIT);
/* Idle the GPU */
kbase_pm_context_idle(kbdev);
return MALI_ERROR_NONE;
}
STATIC INLINE void kbasep_js_ref_permon_check_and_enable_cycle_counter( kbase_device *kbdev, kbase_jd_atom * katom )
{
kbasep_js_device_data *js_devdata;
OSK_ASSERT( kbdev != NULL );
OSK_ASSERT( katom != NULL );
js_devdata = &kbdev->js_data;
if ( katom->atom->core_req & BASE_JD_REQ_PERMON )
{
OSK_ASSERT( js_devdata->runpool_irq.nr_permon_jobs_submitted < S8_MAX );
++js_devdata->runpool_irq.nr_permon_jobs_submitted;
if ( 1 == js_devdata->runpool_irq.nr_permon_jobs_submitted )
{
kbase_reg_write( kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL );
}
}
}
STATIC INLINE void kbasep_js_deref_permon_check_and_disable_cycle_counter( kbase_device *kbdev, kbase_jd_atom * katom )
{
kbasep_js_device_data *js_devdata;
OSK_ASSERT( kbdev != NULL );
OSK_ASSERT( katom != NULL );
js_devdata = &kbdev->js_data;
if ( katom->atom->core_req & BASE_JD_REQ_PERMON )
{
OSK_ASSERT( js_devdata->runpool_irq.nr_permon_jobs_submitted > 0 );
--kbdev->js_data.runpool_irq.nr_permon_jobs_submitted;
if ( 0 == js_devdata->runpool_irq.nr_permon_jobs_submitted )
{
#if BASE_HW_ISSUE_6367 == 0 /* Workaround for issue 6367 requires cycle counter to remain on */
kbase_reg_write( kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL );
#endif
}
}
}
static void pm_callback_power_off(struct kbase_device *kbdev)
{
struct device *dev = kbdev->dev;
int ret = 0, retry = 0;
#if HARD_RESET_AT_POWER_OFF
/* Cause a GPU hard reset to test whether we have actually idled the GPU
* and that we properly reconfigure the GPU on power up.
* Usually this would be dangerous, but if the GPU is working correctly it should
* be completely safe as the GPU should not be active at this point.
* However this is disabled normally because it will most likely interfere with
* bus logging etc.
*/
KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
#endif
if (unlikely(dev->power.disable_depth > 0)) {
kbase_platform_off(kbdev);
} else {
do {
ret = pm_schedule_suspend(dev, RUNTIME_PM_DELAY_TIME);
if (ret != -EAGAIN) {
if (unlikely(ret < 0)) {
pr_err("[mali-midgard] pm_schedule_suspend failed (%d)\n\n", ret);
WARN_ON(1);
}
/* correct status */
break;
}
/* -EAGAIN, repeated attempts for 1s totally */
msleep(50);
} while (++retry < 20);
}
}
void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
u64 *system_time, struct timespec *ts)
{
u32 hi1, hi2;
kbase_pm_request_gpu_cycle_counter(kbdev);
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled
* correctly */
do {
hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI),
NULL);
*cycle_counter = kbase_reg_read(kbdev,
GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI),
NULL);
*cycle_counter |= (((u64) hi1) << 32);
} while (hi1 != hi2);
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled
* correctly */
do {
hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI),
NULL);
*system_time = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TIMESTAMP_LO), NULL);
hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI),
NULL);
*system_time |= (((u64) hi1) << 32);
} while (hi1 != hi2);
/* Record the CPU's idea of current time */
getrawmonotonic(ts);
kbase_pm_release_gpu_cycle_counter(kbdev);
}
static base_jd_event_code kbase_dump_cpu_gpu_time(kbase_jd_atom *katom)
{
kbase_va_region *reg;
osk_phy_addr addr;
u64 pfn;
u32 offset;
char *page;
struct timespec ts;
base_dump_cpu_gpu_counters data;
u64 system_time;
u64 cycle_counter;
mali_addr64 jc = katom->jc;
kbase_context *kctx = katom->kctx;
u32 hi1, hi2;
memset(&data, 0, sizeof(data));
kbase_pm_context_active(kctx->kbdev);
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
do {
hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
cycle_counter = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
cycle_counter |= (((u64)hi1) << 32);
} while (hi1 != hi2);
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
do {
hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
system_time = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_LO), NULL);
hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
system_time |= (((u64)hi1) << 32);
} while (hi1 != hi2);
/* Record the CPU's idea of current time */
getnstimeofday(&ts);
kbase_pm_context_idle(kctx->kbdev);
data.sec = ts.tv_sec;
data.usec = ts.tv_nsec / 1000;
data.system_time = system_time;
data.cycle_counter = cycle_counter;
pfn = jc >> 12;
offset = jc & 0xFFF;
if (offset > 0x1000-sizeof(data))
{
/* Wouldn't fit in the page */
return BASE_JD_EVENT_JOB_CANCELLED;
}
reg = kbase_region_tracker_find_region_enclosing_address(kctx, jc);
if (!reg)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
if (! (reg->flags & KBASE_REG_GPU_WR) )
{
/* Region is not writable by GPU so we won't write to it either */
return BASE_JD_EVENT_JOB_CANCELLED;
}
if (!reg->phy_pages)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
addr = reg->phy_pages[pfn - reg->start_pfn];
if (!addr)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
page = osk_kmap(addr);
if (!page)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
memcpy(page+offset, &data, sizeof(data));
osk_sync_to_cpu(addr+offset, page+offset, sizeof(data));
osk_kunmap(addr, page);
return BASE_JD_EVENT_DONE;
}
STATIC mali_error kbase_instr_hwcnt_enable_internal(kbase_device *kbdev, kbase_context *kctx, kbase_uk_hwcnt_setup *setup)
{
unsigned long flags, pm_flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
kbasep_js_device_data *js_devdata;
u32 irq_mask;
int ret;
u64 shader_cores_needed;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_DEBUG_ASSERT(NULL != setup);
KBASE_DEBUG_ASSERT(NULL == kbdev->hwcnt.suspended_kctx);
shader_cores_needed = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER);
js_devdata = &kbdev->js_data;
/* alignment failure */
if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1)))
goto out_err;
/* Override core availability policy to ensure all cores are available */
kbase_pm_ca_instr_enable(kbdev);
/* Mark the context as active so the GPU is kept turned on */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
/* Request the cores early on synchronously - we'll release them on any errors
* (e.g. instrumentation already active) */
kbase_pm_request_cores_sync(kbdev, MALI_TRUE, shader_cores_needed);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is already enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out_unrequest_cores;
}
/* Enable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | PRFCNT_SAMPLE_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* In use, this context is the owner */
kbdev->hwcnt.kctx = kctx;
/* Remember the dump address so we can reprogram it later */
kbdev->hwcnt.addr = setup->dump_buffer;
/* Remember all the settings for suspend/resume */
if (&kbdev->hwcnt.suspended_state != setup)
memcpy(&kbdev->hwcnt.suspended_state, setup, sizeof(kbdev->hwcnt.suspended_state));
/* Request the clean */
kbdev->hwcnt.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
kbdev->hwcnt.triggered = 0;
/* Clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
ret = queue_work(kbdev->hwcnt.cache_clean_wq, &kbdev->hwcnt.cache_clean_work);
KBASE_DEBUG_ASSERT(ret);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Wait for cacheclean to complete */
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE);
/* Schedule the context in */
kbasep_js_schedule_privileged_ctx(kbdev, kctx);
/* Configure */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_OFF, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), setup->dump_buffer & 0xFFFFFFFF, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), setup->dump_buffer >> 32, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), setup->jm_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), setup->shader_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), setup->l3_cache_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), setup->mmu_l2_bm, kctx);
/* Due to PRLAM-8186 we need to disable the Tiler before we enable the HW counter dump. */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, kctx);
else
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, kctx);
/* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
/* GPU is being reset */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
}
kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.triggered = 1;
wake_up(&kbdev->hwcnt.wait);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
err = MALI_ERROR_NONE;
dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx);
return err;
out_unrequest_cores:
kbase_pm_unrequest_cores(kbdev, MALI_TRUE, shader_cores_needed);
kbase_pm_context_idle(kbdev);
out_err:
return err;
}
void kbase_backend_gpuprops_get(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
int i;
/* Fill regdump with the content of the relevant registers */
regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID), NULL);
regdump->l2_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_FEATURES), NULL);
regdump->suspend_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SUSPEND_SIZE), NULL);
regdump->tiler_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_FEATURES), NULL);
regdump->mem_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MEM_FEATURES), NULL);
regdump->mmu_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MMU_FEATURES), NULL);
regdump->as_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(AS_PRESENT), NULL);
regdump->js_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_PRESENT), NULL);
for (i = 0; i < GPU_MAX_JOB_SLOTS; i++)
regdump->js_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_FEATURES_REG(i)), NULL);
for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
regdump->texture_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)), NULL);
regdump->thread_max_threads = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_THREADS), NULL);
regdump->thread_max_workgroup_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE),
NULL);
regdump->thread_max_barrier_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE), NULL);
regdump->thread_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_FEATURES), NULL);
regdump->shader_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_LO), NULL);
regdump->shader_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_HI), NULL);
regdump->tiler_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_LO), NULL);
regdump->tiler_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_HI), NULL);
regdump->l2_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_LO), NULL);
regdump->l2_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_HI), NULL);
}
/**
* @brief Disable HW counters collection
*
* Note: might sleep, waiting for an ongoing dump to complete
*/
mali_error kbase_instr_hwcnt_disable_sec(struct kbase_context *kctx)
{
unsigned long flags, pm_flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
u32 irq_mask;
struct kbase_device *kbdev;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
/* MALI_SEC 140925 */
flush_work(&kbdev->hwcnt.cache_clean_work);
while (1) {
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is not enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out;
}
if (kbdev->hwcnt.kctx != kctx) {
/* Instrumentation has been setup for another context */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out;
}
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE)
break;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Ongoing dump/setup - wait for its completion */
if (wait_event_timeout(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0, kbdev->hwcnt.timeout) == 0)
kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
}
kbdev->hwcnt.state = KBASE_INSTR_STATE_DISABLED;
kbdev->hwcnt.triggered = 0;
/* Disable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* Disable the counters */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx);
kbdev->hwcnt.kctx = NULL;
kbdev->hwcnt.addr = 0ULL;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
err = MALI_ERROR_NONE;
out:
kbdev->hwcnt.trig_exception = 0;
return err;
}
STATIC void kbase_gpuprops_dump_registers(kbase_device *kbdev, kbase_gpuprops_regdump *regdump)
{
int i;
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_DEBUG_ASSERT(NULL != regdump);
/* Fill regdump with the content of the relevant registers */
regdump->gpu_id = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
regdump->l2_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_FEATURES));
regdump->l3_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_FEATURES));
regdump->tiler_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_FEATURES));
regdump->mem_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(MEM_FEATURES));
regdump->mmu_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(MMU_FEATURES));
regdump->as_present = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(AS_PRESENT));
regdump->js_present = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(JS_PRESENT));
for (i = 0; i < MIDG_MAX_JOB_SLOTS; i++)
regdump->js_features[i] = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(JS_FEATURES_REG(i)));
for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
regdump->texture_features[i] = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)));
regdump->thread_max_threads = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_THREADS));
regdump->thread_max_workgroup_size = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE));
regdump->thread_max_barrier_size = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE));
regdump->thread_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_FEATURES));
regdump->shader_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_LO));
regdump->shader_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_HI));
regdump->tiler_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_LO));
regdump->tiler_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_HI));
regdump->l2_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_LO));
regdump->l2_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_HI));
regdump->l3_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_PRESENT_LO));
regdump->l3_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_PRESENT_HI));
}
static void pm_callback_power_off(struct kbase_device *kbdev)
{
unsigned int uiCurrentFreqCount;
volatile int polling_count = 100000;
volatile int i = 0;
unsigned int code;
/// 1. Delay 0.01ms before power off
for (i=0; i < DELAY_LOOP_COUNT;i++);
if (DELAY_LOOP_COUNT != i)
{
pr_debug("[MALI] power off delay error!\n");
}
/// 2. Polling the MFG_DEBUG_REG for checking GPU IDLE before MTCMOS power off (0.1ms)
MFG_WRITE32(0x3, MFG_DEBUG_CTRL_REG);
do {
/// 0x13000184[2]
/// 1'b1: bus idle
/// 1'b0: bus busy
if (MFG_READ32(MFG_DEBUG_STAT_REG) & MFG_BUS_IDLE_BIT)
{
/// pr_debug("[MALI]MFG BUS already IDLE! Ready to power off, %d\n", polling_count);
break;
}
} while (polling_count--);
if (polling_count <=0)
{
pr_debug("[MALI]!!!!MFG(GPU) subsys is still BUSY!!!!!, polling_count=%d\n", polling_count);
}
#if HARD_RESET_AT_POWER_OFF
/* Cause a GPU hard reset to test whether we have actually idled the GPU
* and that we properly reconfigure the GPU on power up.
* Usually this would be dangerous, but if the GPU is working correctly it should
* be completely safe as the GPU should not be active at this point.
* However this is disabled normally because it will most likely interfere with
* bus logging etc.
*/
//KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
#endif
/// Polling the MFG_DEBUG_REG for checking GPU IDLE before MTCMOS power off (0.1ms)
MFG_WRITE32(0x3, MFG_DEBUG_CTRL_REG);
do {
/// 0x13000184[2]
/// 1'b1: bus idle
/// 1'b0: bus busy
if (MFG_READ32(MFG_DEBUG_STAT_REG) & MFG_BUS_IDLE_BIT)
{
/// pr_debug("[MALI]MFG BUS already IDLE! Ready to power off, %d\n", polling_count);
break;
}
} while (polling_count--);
if (polling_count <=0)
{
pr_debug("[MALI]!!!!MFG(GPU) subsys is still BUSY!!!!!, polling_count=%d\n", polling_count);
}
g_power_status = 0; // the power status is "power off".
g_power_off_gpu_freq_idx = mt_gpufreq_get_cur_freq_index(); // record current freq. index.
//pr_debug("MALI: GPU power off freq idx : %d\n",g_power_off_gpu_freq_idx );
#if 1
uiCurrentFreqCount = mt_gpufreq_get_dvfs_table_num(); // get freq. table size
mt_gpufreq_target(uiCurrentFreqCount-1); // set gpu to lowest freq.
#endif
code = mt_get_chip_hw_code();
/* MTK clock modified */
if (0x321 == code) {
// do something for Denali-1(6735)
#ifdef CONFIG_MTK_CLKMGR
disable_clock( MT_CG_MFG_BG3D, "GPU");
disable_clock( MT_CG_DISP0_SMI_COMMON, "GPU");
#else
clk_disable_unprepare(kbdev->clk_mfg);
clk_disable_unprepare(kbdev->clk_mfg_scp);
clk_disable_unprepare(kbdev->clk_smi_common);
clk_disable_unprepare(kbdev->clk_display_scp);
#endif
} else if (0x335 == code) {
// do something for Denali-2(6735M)
#ifdef CONFIG_MTK_CLKMGR
disable_clock( MT_CG_MFG_BG3D, "GPU");
disable_clock( MT_CG_DISP0_SMI_COMMON, "GPU");
#endif /* CONFIG_MTK_CLKMGR */
} else if (0x337 == code) {
// do something for Denali-3(6753)
#ifdef CONFIG_MTK_CLKMGR
disable_clock( MT_CG_MFG_BG3D, "GPU");
disable_clock( MT_CG_DISP0_SMI_COMMON, "GPU");
#endif /* CONFIG_MTK_CLKMGR */
} else {
// unknown chip ID, error !!
#ifdef CONFIG_MTK_CLKMGR
disable_clock( MT_CG_MFG_BG3D, "GPU");
disable_clock( MT_CG_DISP0_SMI_COMMON, "GPU");
#endif /* CONFIG_MTK_CLKMGR */
}
mt_gpufreq_voltage_enable_set(0);
}
static base_jd_event_code kbase_dump_cpu_gpu_time(kbase_context *kctx, mali_addr64 jc)
{
kbase_va_region *reg;
osk_phy_addr addr;
u64 pfn;
u32 offset;
char *page;
osk_timeval tv;
base_dump_cpu_gpu_counters data;
u64 system_time;
u64 cycle_counter;
u32 hi1, hi2;
OSK_MEMSET(&data, 0, sizeof(data));
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
do {
hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
cycle_counter = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
cycle_counter |= (((u64)hi1) << 32);
} while (hi1 != hi2);
/* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
do {
hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
system_time = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_LO), NULL);
hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
system_time |= (((u64)hi1) << 32);
} while (hi1 != hi2);
/* Record the CPU's idea of current time */
osk_gettimeofday(&tv);
data.sec = tv.tv_sec;
data.usec = tv.tv_usec;
data.system_time = system_time;
data.cycle_counter = cycle_counter;
pfn = jc >> 12;
offset = jc & 0xFFF;
if (offset > 0x1000-sizeof(data))
{
/* Wouldn't fit in the page */
return BASE_JD_EVENT_JOB_CANCELLED;
}
reg = kbase_region_lookup(kctx, jc);
if (!reg)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
if (! (reg->flags & KBASE_REG_GPU_RW) )
{
/* Region is not writable by GPU so we won't write to it either */
return BASE_JD_EVENT_JOB_CANCELLED;
}
if (!reg->phy_pages)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
addr = reg->phy_pages[pfn - reg->start_pfn];
if (!addr)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
page = osk_kmap(addr);
if (!page)
{
return BASE_JD_EVENT_JOB_CANCELLED;
}
memcpy(page+offset, &data, sizeof(data));
osk_kunmap(addr, page);
return BASE_JD_EVENT_DONE;
}
void kbasep_js_try_schedule_head_ctx( kbase_device *kbdev )
{
kbasep_js_device_data *js_devdata;
mali_bool has_kctx;
kbase_context *head_kctx;
kbasep_js_kctx_info *js_kctx_info;
mali_bool is_runpool_full;
OSK_ASSERT( kbdev != NULL );
js_devdata = &kbdev->js_data;
/* Make a speculative check on the Run Pool - this MUST be repeated once
* we've obtained a context from the queue and reobtained the Run Pool
* lock */
osk_mutex_lock( &js_devdata->runpool_mutex );
is_runpool_full = (mali_bool)( js_devdata->nr_contexts_running >= kbdev->nr_address_spaces );
osk_mutex_unlock( &js_devdata->runpool_mutex );
if ( is_runpool_full != MALI_FALSE )
{
/* No free address spaces - nothing to do */
return;
}
/* Grab the context off head of queue - if there is one */
osk_mutex_lock( &js_devdata->queue_mutex );
has_kctx = kbasep_js_policy_dequeue_head_ctx( &js_devdata->policy, &head_kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
if ( has_kctx == MALI_FALSE )
{
/* No ctxs to run - nothing to do */
return;
}
js_kctx_info = &head_kctx->jctx.sched_info;
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Dequeue Context %p", head_kctx );
/*
* Atomic transaction on the Context and Run Pool begins
*/
osk_mutex_lock( &js_kctx_info->ctx.jsctx_mutex );
osk_mutex_lock( &js_devdata->runpool_mutex );
/* Re-check to see if the Run Pool is full */
is_runpool_full = (mali_bool)( js_devdata->nr_contexts_running >= kbdev->nr_address_spaces );
if ( is_runpool_full != MALI_FALSE )
{
/* No free address spaces - roll back the transaction so far and return */
osk_mutex_unlock( &js_devdata->runpool_mutex );
/* Only requeue if not dying - which might occur through zapping-whilst-scheduling */
if ( js_kctx_info->ctx.is_dying == MALI_FALSE )
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Transaction rollback: Requeue Context %p", head_kctx );
osk_mutex_lock( &js_devdata->queue_mutex );
kbasep_js_policy_enqueue_ctx( &js_devdata->policy, head_kctx );
osk_mutex_unlock( &js_devdata->queue_mutex );
}
else
{
OSK_PRINT_INFO(OSK_BASE_JM, "JS: Transaction rollback: Context %p is dying. Kill remaining jobs and pm-idle ctx", head_kctx );
OSK_ASSERT( js_kctx_info->ctx.nr_jobs > 0 );
/* Notify PM that a context has gone idle */
kbase_pm_context_idle(kbdev);
/* Kill all the jobs present (call kbase_jd_cancel on all jobs) */
kbasep_js_policy_kill_all_ctx_jobs( &js_devdata->policy, head_kctx );
/* Nothing more to be done to kill the context here, kbase_jd_zap_context
* waits for all jobs to be cancelled */
}
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
return;
}
OSK_PRINT_INFO(OSK_BASE_JM, "JS: RunPool Add Context %p", head_kctx );
/* update book-keeping info */
js_kctx_info->ctx.is_scheduled = MALI_TRUE;
++(js_devdata->nr_contexts_running);
/* Cause any future waiter-on-termination to wait until the context is
* descheduled */
osk_waitq_clear( &js_kctx_info->ctx.not_scheduled_waitq );
/* Do all the necessaries to pick the address space (inc. update book-keeping info)
* Add the context to the Run Pool, and allow it to run jobs */
assign_and_activate_kctx_addr_space( kbdev, head_kctx );
/* Check and setup HW counters dumping */
osk_spinlock_lock(&kbdev->hwcnt_lock);
osk_spinlock_irq_lock(&js_devdata->runpool_irq.lock);
if (head_kctx == kbdev->hwcnt_context &&
kbdev->hwcnt_is_setup == MALI_FALSE)
{
/* Setup the base address */
#if BASE_HW_ISSUE_8186
u32 val;
/* Save and clear PRFCNT_TILER_EN */
val = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), head_kctx);
if(0 != val)
{
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, head_kctx);
}
/* Update PRFCNT_CONFIG with TILER_EN = 0 */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (head_kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, head_kctx);
/* Restore PRFCNT_TILER_EN */
if(0 != val)
{
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), val, head_kctx);
}
#else
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (head_kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, head_kctx);
#endif
/* Prevent the context to be scheduled out */
kbasep_js_runpool_retain_ctx_nolock(kbdev, head_kctx);
kbdev->hwcnt_is_setup = MALI_TRUE;
}
osk_spinlock_irq_unlock(&js_devdata->runpool_irq.lock);
osk_spinlock_unlock(&kbdev->hwcnt_lock);
/* Try to run the next job, in case this context has jobs that match the
* job slot requirements, but none of the other currently running contexts
* do */
kbasep_js_try_run_next_job( kbdev );
/* Transaction complete */
osk_mutex_unlock( &js_devdata->runpool_mutex );
osk_mutex_unlock( &js_kctx_info->ctx.jsctx_mutex );
/* Note: after this point, the context could potentially get scheduled out immediately */
}
STATIC void kbase_gpuprops_dump_registers(kbase_device * kbdev, kbase_gpuprops_regdump * regdump)
{
int i;
OSK_ASSERT(NULL != kbdev);
OSK_ASSERT(NULL != regdump);
/* Fill regdump with the content of the relevant registers */
regdump->gpu_id = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
regdump->l2_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_FEATURES));
regdump->l3_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_FEATURES));
regdump->tiler_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_FEATURES));
regdump->mem_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(MEM_FEATURES));
regdump->mmu_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(MMU_FEATURES));
regdump->as_present = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(AS_PRESENT));
regdump->js_present = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(JS_PRESENT));
for(i = 0; i < MIDG_MAX_JOB_SLOTS; i++)
{
regdump->js_features[i] = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(JS_FEATURES_REG(i)));
}
for(i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
{
regdump->texture_features[i] = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)));
}
regdump->shader_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_LO));
regdump->shader_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_HI));
regdump->tiler_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_LO));
regdump->tiler_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_HI));
regdump->l2_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_LO));
regdump->l2_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_HI));
regdump->l3_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_PRESENT_LO));
regdump->l3_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_PRESENT_HI));
}
static void pm_callback_power_off(struct kbase_device *kbdev)
{
unsigned int uiCurrentFreqCount;
volatile int polling_count = 100000;
volatile int i = 0;
struct mtk_config *config;
if (!kbdev) {
pr_alert("MALI: input parameter is NULL \n");
}
config = (struct mtk_config *)kbdev->mtk_config;
if (!config) {
pr_alert("MALI: mtk_config is NULL \n");
}
/// 1. Delay 0.01ms before power off
for (i=0; i < DELAY_LOOP_COUNT;i++);
if (DELAY_LOOP_COUNT != i)
{
pr_warn("[MALI] power off delay error!\n");
}
/// 2. Polling the MFG_DEBUG_REG for checking GPU IDLE before MTCMOS power off (0.1ms)
MFG_WRITE32(0x3, MFG_DEBUG_CTRL_REG);
do {
/// 0x13000184[2]
/// 1'b1: bus idle
/// 1'b0: bus busy
if (MFG_READ32(MFG_DEBUG_STAT_REG) & MFG_BUS_IDLE_BIT)
{
/// printk("[MALI]MFG BUS already IDLE! Ready to power off, %d\n", polling_count);
break;
}
} while (polling_count--);
if (polling_count <=0)
{
pr_warn("[MALI]!!!!MFG(GPU) subsys is still BUSY!!!!!, polling_count=%d\n", polling_count);
}
#if HARD_RESET_AT_POWER_OFF
/* Cause a GPU hard reset to test whether we have actually idled the GPU
* and that we properly reconfigure the GPU on power up.
* Usually this would be dangerous, but if the GPU is working correctly it should
* be completely safe as the GPU should not be active at this point.
* However this is disabled normally because it will most likely interfere with
* bus logging etc.
*/
//KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
/// Polling the MFG_DEBUG_REG for checking GPU IDLE before MTCMOS power off (0.1ms)
MFG_WRITE32(0x3, MFG_DEBUG_CTRL_REG);
do {
/// 0x13000184[2]
/// 1'b1: bus idle
/// 1'b0: bus busy
if (MFG_READ32(MFG_DEBUG_STAT_REG) & MFG_BUS_IDLE_BIT)
{
/// printk("[MALI]MFG BUS already IDLE! Ready to power off, %d\n", polling_count);
break;
}
} while (polling_count--);
if (polling_count <=0)
{
printk("[MALI]!!!!MFG(GPU) subsys is still BUSY!!!!!, polling_count=%d\n", polling_count);
}
g_power_off_gpu_freq_idx = mt_gpufreq_get_cur_freq_index(); // record current freq. index.
//printk("MALI: GPU power off freq idx : %d\n",g_power_off_gpu_freq_idx );
#if 1
uiCurrentFreqCount = mt_gpufreq_get_dvfs_table_num(); // get freq. table size
mt_gpufreq_target(uiCurrentFreqCount-1); // set gpu to lowest freq.
#endif
/* MTK clock modified */
#ifdef CONFIG_MTK_CLKMGR
disable_clock( MT_CG_MFG_BG3D, "GPU");
disable_clock( MT_CG_DISP0_SMI_COMMON, "GPU");
#endif
if(mt6325_upmu_get_swcid() >= PMIC6325_E3_CID_CODE)
{
mt_gpufreq_voltage_enable_set(0);
}
#ifdef ENABLE_COMMON_DVFS
ged_dvfs_gpu_clock_switch_notify(0);
#endif
mtk_set_vgpu_power_on_flag(MTK_VGPU_POWER_OFF); // the power status is "power off".
#endif
}
bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx,
int reg_range)
{
int i, j;
int offset = 0;
int slot_number;
int as_number;
if (kctx->reg_dump == NULL)
return false;
slot_number = kctx->kbdev->gpu_props.num_job_slots;
as_number = kctx->kbdev->gpu_props.num_address_spaces;
/* get the GPU control registers*/
for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
GPU_CONTROL_REG(gpu_control_reg_snapshot[i]);
offset += 2;
}
/* get the Job control registers*/
for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
JOB_CONTROL_REG(job_control_reg_snapshot[i]);
offset += 2;
}
/* get the Job Slot registers*/
for (j = 0; j < slot_number; j++) {
for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
JOB_SLOT_REG(j, job_slot_reg_snapshot[i]);
offset += 2;
}
}
/* get the MMU registers*/
for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]);
offset += 2;
}
/* get the Address space registers*/
for (j = 0; j < as_number; j++) {
for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
MMU_AS_REG(j, as_reg_snapshot[i]);
offset += 2;
}
}
WARN_ON(offset >= (reg_range*2/4));
/* set the termination flag*/
kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG;
kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG;
dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n",
offset);
return true;
}
/**
* @brief Disable HW counters collection
*
* Note: might sleep, waiting for an ongoing dump to complete
*/
mali_error kbase_instr_hwcnt_disable(kbase_context *kctx)
{
unsigned long flags, pm_flags;
mali_error err = MALI_ERROR_FUNCTION_FAILED;
u32 irq_mask;
kbase_device *kbdev;
KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(NULL != kbdev);
while (1) {
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is not enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out;
}
if (kbdev->hwcnt.kctx != kctx) {
/* Instrumentation has been setup for another context */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out;
}
if (kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE)
break;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Ongoing dump/setup - wait for its completion */
wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
}
kbdev->hwcnt.state = KBASE_INSTR_STATE_DISABLED;
kbdev->hwcnt.triggered = 0;
/* Disable interrupt */
spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
/* Disable the counters */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx);
kbdev->hwcnt.kctx = NULL;
kbdev->hwcnt.addr = 0ULL;
kbase_pm_ca_instr_disable(kbdev);
kbase_pm_unrequest_cores(kbdev, MALI_TRUE, kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER));
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Release the context. This had its own Power Manager Active reference */
kbasep_js_release_privileged_ctx(kbdev, kctx);
/* Also release our Power Manager Active reference */
kbase_pm_context_idle(kbdev);
dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
err = MALI_ERROR_NONE;
out:
return err;
}
