mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence)
{
mali_timeline_point point;
struct mali_soft_job_system *system;
MALI_DEBUG_ASSERT_POINTER(job);
MALI_DEBUG_ASSERT_POINTER(fence);
MALI_DEBUG_ASSERT_POINTER(job->system);
system = job->system;
MALI_DEBUG_ASSERT_POINTER(system->session);
MALI_DEBUG_ASSERT_POINTER(system->session->timeline_system);
mali_soft_job_system_lock(system);
MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state);
job->state = MALI_SOFT_JOB_STATE_STARTED;
mali_soft_job_system_unlock(system);
MALI_DEBUG_PRINT(4, ("Mali Soft Job: starting soft job %u (0x%08X)\n", job->id, job));
mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_SOFT, fence, job);
point = mali_timeline_system_add_tracker(system->session->timeline_system, &job->tracker, MALI_TIMELINE_SOFT);
return point;
}
/**
* Creates a sync fence tracker and a sync fence. Adds sync fence tracker to Timeline system and
* returns sync fence. The sync fence will be signaled when the sync fence tracker is activated.
*
* @param timeline Timeline.
* @param point Point on timeline.
* @return Sync fence that will be signaled when tracker is activated.
*/
static struct sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point)
{
struct mali_timeline_sync_fence_tracker *sync_fence_tracker;
struct sync_fence *sync_fence;
struct mali_timeline_fence fence;
MALI_DEBUG_ASSERT_POINTER(timeline);
MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point);
/* Allocate sync fence tracker. */
sync_fence_tracker = _mali_osk_calloc(1, sizeof(struct mali_timeline_sync_fence_tracker));
if (NULL == sync_fence_tracker) {
MALI_PRINT_ERROR(("Mali Timeline: sync_fence_tracker allocation failed\n"));
return NULL;
}
/* Create sync flag. */
MALI_DEBUG_ASSERT_POINTER(timeline->sync_tl);
sync_fence_tracker->flag = mali_sync_flag_create(timeline->sync_tl, point);
if (NULL == sync_fence_tracker->flag) {
MALI_PRINT_ERROR(("Mali Timeline: sync_flag creation failed\n"));
_mali_osk_free(sync_fence_tracker);
return NULL;
}
/* Create sync fence from sync flag. */
sync_fence = mali_sync_flag_create_fence(sync_fence_tracker->flag);
if (NULL == sync_fence) {
MALI_PRINT_ERROR(("Mali Timeline: sync_fence creation failed\n"));
mali_sync_flag_put(sync_fence_tracker->flag);
_mali_osk_free(sync_fence_tracker);
return NULL;
}
/* Setup fence for tracker. */
_mali_osk_memset(&fence, 0, sizeof(struct mali_timeline_fence));
fence.sync_fd = -1;
fence.points[timeline->id] = point;
/* Finally, add the tracker to Timeline system. */
mali_timeline_tracker_init(&sync_fence_tracker->tracker, MALI_TIMELINE_TRACKER_SYNC, &fence, sync_fence_tracker);
point = mali_timeline_system_add_tracker(timeline->system, &sync_fence_tracker->tracker, MALI_TIMELINE_NONE);
MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point);
return sync_fence;
}
struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker)
{
struct mali_gp_job *job;
u32 perf_counter_flag;
job = _mali_osk_malloc(sizeof(struct mali_gp_job));
if (NULL != job) {
job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_FINISHED, sizeof(_mali_uk_gp_job_finished_s));
if (NULL == job->finished_notification) {
_mali_osk_free(job);
return NULL;
}
job->oom_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s));
if (NULL == job->oom_notification) {
_mali_osk_notification_delete(job->finished_notification);
_mali_osk_free(job);
return NULL;
}
if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) {
_mali_osk_notification_delete(job->finished_notification);
_mali_osk_notification_delete(job->oom_notification);
_mali_osk_free(job);
return NULL;
}
perf_counter_flag = mali_gp_job_get_perf_counter_flag(job);
/* case when no counters came from user space
* so pass the debugfs / DS-5 provided global ones to the job object */
if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) ||
(perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) {
mali_gp_job_set_perf_counter_src0(job, mali_gp_job_get_gp_counter_src0());
mali_gp_job_set_perf_counter_src1(job, mali_gp_job_get_gp_counter_src1());
}
_mali_osk_list_init(&job->list);
job->session = session;
job->id = id;
job->heap_current_addr = job->uargs.frame_registers[4];
job->perf_counter_value0 = 0;
job->perf_counter_value1 = 0;
job->pid = _mali_osk_get_pid();
job->tid = _mali_osk_get_tid();
job->pp_tracker = pp_tracker;
if (NULL != job->pp_tracker) {
/* Take a reference on PP job's tracker that will be released when the GP
job is done. */
mali_timeline_system_tracker_get(session->timeline_system, pp_tracker);
}
mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_GP, NULL, job);
mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence));
return job;
}
return NULL;
}
struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session,
_mali_uk_pp_start_job_s __user *uargs, u32 id)
{
struct mali_pp_job *job;
u32 perf_counter_flag;
job = _mali_osk_calloc(1, sizeof(struct mali_pp_job));
if (NULL != job) {
if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) {
goto fail;
}
if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) {
MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n"));
goto fail;
}
if (!mali_pp_job_use_no_notification(job)) {
job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s));
if (NULL == job->finished_notification) goto fail;
}
perf_counter_flag = mali_pp_job_get_perf_counter_flag(job);
/* case when no counters came from user space
* so pass the debugfs / DS-5 provided global ones to the job object */
if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) ||
(perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) {
u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count);
/* These counters apply for all virtual jobs, and where no per sub job counter is specified */
job->uargs.perf_counter_src0 = pp_counter_src0;
job->uargs.perf_counter_src1 = pp_counter_src1;
/* We only copy the per sub job array if it is enabled with at least one counter */
if (0 < sub_job_count) {
job->perf_counter_per_sub_job_count = sub_job_count;
_mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0));
_mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1));
}
}
_mali_osk_list_init(&job->list);
job->session = session;
_mali_osk_list_init(&job->session_list);
job->id = id;
job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1;
job->pid = _mali_osk_get_pid();
job->tid = _mali_osk_get_tid();
job->num_memory_cookies = job->uargs.num_memory_cookies;
if (job->num_memory_cookies > 0) {
u32 size;
u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies;
if (job->uargs.num_memory_cookies > session->descriptor_mapping->current_nr_mappings) {
MALI_PRINT_ERROR(("Mali PP job: Too many memory cookies specified in job object\n"));
goto fail;
}
size = sizeof(*memory_cookies) * job->num_memory_cookies;
job->memory_cookies = _mali_osk_malloc(size);
if (NULL == job->memory_cookies) {
MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size));
goto fail;
}
if (0 != _mali_osk_copy_from_user(job->memory_cookies, memory_cookies, size)) {
MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size));
goto fail;
}
#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
job->num_dma_bufs = job->num_memory_cookies;
job->dma_bufs = _mali_osk_calloc(job->num_dma_bufs, sizeof(struct mali_dma_buf_attachment *));
if (NULL == job->dma_bufs) {
MALI_PRINT_ERROR(("Mali PP job: Failed to allocate dma_bufs array!\n"));
goto fail;
}
#endif
}
/* Prepare DMA command buffer to start job, if it is virtual. */
if (mali_pp_job_is_virtual_group_job(job)) {
struct mali_pp_core *core;
_mali_osk_errcode_t err = mali_dma_get_cmd_buf(&job->dma_cmd_buf);
if (_MALI_OSK_ERR_OK != err) {
MALI_PRINT_ERROR(("Mali PP job: Failed to allocate DMA command buffer\n"));
goto fail;
}
core = mali_pp_scheduler_get_virtual_pp();
MALI_DEBUG_ASSERT_POINTER(core);
mali_pp_job_dma_cmd_prepare(core, job, 0, &job->dma_cmd_buf);
}
if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) {
/* Not a valid job. */
goto fail;
}
mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job);
mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence));
return job;
}
fail:
if (NULL != job) {
mali_pp_job_delete(job);
}
return NULL;
}
/**
* Check if fence has been signaled.
*
* @param system Timeline system.
* @param fence Timeline fence.
* @return MALI_TRUE if fence is signaled, MALI_FALSE if not.
*/
static mali_bool mali_timeline_fence_wait_check_status(struct mali_timeline_system *system, struct mali_timeline_fence *fence)
{
int i;
u32 tid = _mali_osk_get_tid();
mali_bool ret = MALI_TRUE;
#if defined(CONFIG_SYNC)
struct sync_fence *sync_fence = NULL;
#endif
MALI_DEBUG_ASSERT_POINTER(system);
MALI_DEBUG_ASSERT_POINTER(fence);
mali_spinlock_reentrant_wait(system->spinlock, tid);
for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
struct mali_timeline *timeline;
mali_timeline_point point;
point = fence->points[i];
if (likely(MALI_TIMELINE_NO_POINT == point)) {
/* Fence contains no point on this timeline. */
continue;
}
timeline = system->timelines[i];
MALI_DEBUG_ASSERT_POINTER(timeline);
if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", point, timeline->point_oldest, timeline->point_next));
}
if (!mali_timeline_is_point_released(timeline, point)) {
ret = MALI_FALSE;
goto exit;
}
}
#if defined(CONFIG_SYNC)
if (-1 != fence->sync_fd) {
sync_fence = sync_fence_fdget(fence->sync_fd);
if (likely(NULL != sync_fence)) {
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
if (0 == sync_fence->status) {
#else
if (0 == atomic_read(&sync_fence->status)) {
#endif
ret = MALI_FALSE;
}
} else {
MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", fence->sync_fd));
}
}
#endif /* defined(CONFIG_SYNC) */
exit:
mali_spinlock_reentrant_signal(system->spinlock, tid);
#if defined(CONFIG_SYNC)
if (NULL != sync_fence) {
sync_fence_put(sync_fence);
}
#endif /* defined(CONFIG_SYNC) */
return ret;
}
mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout)
{
struct mali_timeline_fence_wait_tracker *wait;
mali_timeline_point point;
mali_bool ret;
MALI_DEBUG_ASSERT_POINTER(system);
MALI_DEBUG_ASSERT_POINTER(fence);
MALI_DEBUG_PRINT(4, ("Mali Timeline: wait on fence\n"));
if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY == timeout) {
return mali_timeline_fence_wait_check_status(system, fence);
}
wait = mali_timeline_fence_wait_tracker_alloc();
if (unlikely(NULL == wait)) {
MALI_PRINT_ERROR(("Mali Timeline: failed to allocate data for fence wait\n"));
return MALI_FALSE;
}
wait->activated = MALI_FALSE;
wait->system = system;
/* Initialize refcount to two references. The reference first will be released by this
* function after the wait is over. The second reference will be released when the tracker
* is activated. */
_mali_osk_atomic_init(&wait->refcount, 2);
/* Add tracker to timeline system, but not to a timeline. */
mali_timeline_tracker_init(&wait->tracker, MALI_TIMELINE_TRACKER_WAIT, fence, wait);
point = mali_timeline_system_add_tracker(system, &wait->tracker, MALI_TIMELINE_NONE);
MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point);
MALI_IGNORE(point);
/* Wait for the tracker to be activated or time out. */
if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER == timeout) {
_mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait);
} else {
_mali_osk_wait_queue_wait_event_timeout(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait, timeout);
}
ret = wait->activated;
if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) {
mali_timeline_fence_wait_tracker_free(wait);
}
return ret;
}
void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *wait)
{
mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
MALI_DEBUG_ASSERT_POINTER(wait);
MALI_DEBUG_ASSERT_POINTER(wait->system);
MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for fence wait tracker\n"));
MALI_DEBUG_ASSERT(MALI_FALSE == wait->activated);
wait->activated = MALI_TRUE;
_mali_osk_wait_queue_wake_up(wait->system->wait_queue);
/* Nothing can wait on this tracker, so nothing to schedule after release. */
schedule_mask = mali_timeline_tracker_release(&wait->tracker);
MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask);
MALI_IGNORE(schedule_mask);
if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) {
mali_timeline_fence_wait_tracker_free(wait);
}
}
