struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session) { struct mali_soft_job_system *system; MALI_DEBUG_ASSERT_POINTER(session); system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system)); if (NULL == system) { return NULL; } system->session = session; system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER); if (NULL == system->lock) { mali_soft_job_system_destroy(system); return NULL; } system->lock_owner = 0; system->last_job_id = 0; _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used)); return system; }
static int _mali_gp_add_varying_allocations(struct mali_session_data *session, struct mali_gp_job *job, u32 *alloc, u32 num) { int i = 0; struct mali_gp_allocation_node *alloc_node; mali_mem_allocation *mali_alloc = NULL; struct mali_vma_node *mali_vma_node = NULL; for (i = 0 ; i < num ; i++) { MALI_DEBUG_ASSERT(alloc[i]); alloc_node = _mali_osk_calloc(1, sizeof(struct mali_gp_allocation_node)); if (alloc_node) { INIT_LIST_HEAD(&alloc_node->node); /* find mali allocation structure by vaddress*/ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, alloc[i], 0); if (likely(mali_vma_node)) { mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); MALI_DEBUG_ASSERT(alloc[i] == mali_vma_node->vm_node.start); } else { MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,can't find allocation %d by address =0x%x, num=%d\n", i, alloc[i], num)); MALI_DEBUG_ASSERT(0); } alloc_node->alloc = mali_alloc; /* add to gp job varying alloc list*/ list_move(&alloc_node->node, &job->varying_alloc); } else goto fail;
mali_descriptor_mapping * mali_descriptor_mapping_create(int init_entries, int max_entries) { mali_descriptor_mapping * map = _mali_osk_calloc(1, sizeof(mali_descriptor_mapping)); init_entries = MALI_PAD_INT(init_entries); max_entries = MALI_PAD_INT(max_entries); if (NULL != map) { map->table = descriptor_table_alloc(init_entries); if (NULL != map->table) { #if !USING_MMU map->lock = _mali_osk_lock_init( (_mali_osk_lock_flags_t)(_MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_READERWRITER | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE), 0, 20); #else map->lock = _mali_osk_lock_init( (_mali_osk_lock_flags_t)(_MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_READERWRITER | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE), 0, 116); #endif if (NULL != map->lock) { _mali_osk_set_nonatomic_bit(0, map->table->usage); /* reserve bit 0 to prevent NULL/zero logic to kick in */ map->max_nr_mappings_allowed = max_entries; map->current_nr_mappings = init_entries; return map; } descriptor_table_free(map->table); } _mali_osk_free(map); } return NULL; }
struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session) { u32 i; struct mali_soft_job_system *system; struct mali_soft_job *job; MALI_DEBUG_ASSERT_POINTER(session); system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system)); if (NULL == system) { return NULL; } system->session = session; system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER); if (NULL == system->lock) { mali_soft_job_system_destroy(system); return NULL; } system->lock_owner = 0; _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_free)); _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used)); for (i = 0; i < MALI_MAX_NUM_SOFT_JOBS; ++i) { job = &(system->jobs[i]); _mali_osk_list_add(&(job->system_list), &(system->jobs_free)); job->system = system; job->state = MALI_SOFT_JOB_STATE_FREE; job->id = i; } return system; }
mali_descriptor_mapping * mali_descriptor_mapping_create(int init_entries, int max_entries) { mali_descriptor_mapping * map = _mali_osk_calloc(1, sizeof(mali_descriptor_mapping)); init_entries = MALI_PAD_INT(init_entries); max_entries = MALI_PAD_INT(max_entries); if (NULL != map) { map->table = descriptor_table_alloc(init_entries); if (NULL != map->table) { map->lock = _mali_osk_lock_init( (_mali_osk_lock_flags_t)(_MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_READERWRITER | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE), 0, _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP); if (NULL != map->lock) { _mali_osk_set_nonatomic_bit(0, map->table->usage); map->max_nr_mappings_allowed = max_entries; map->current_nr_mappings = init_entries; return map; } descriptor_table_free(map->table); } _mali_osk_free(map); } return NULL; }
struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual) { struct mali_mmu_core* mmu = NULL; MALI_DEBUG_ASSERT_POINTER(resource); MALI_DEBUG_PRINT(2, ("Mali MMU: Creating Mali MMU: %s\n", resource->description)); mmu = _mali_osk_calloc(1,sizeof(struct mali_mmu_core)); if (NULL != mmu) { if (_MALI_OSK_ERR_OK == mali_hw_core_create(&mmu->hw_core, resource, MALI_MMU_REGISTERS_SIZE)) { if (_MALI_OSK_ERR_OK == mali_group_add_mmu_core(group, mmu)) { if (is_virtual) { /* Skip reset and IRQ setup for virtual MMU */ return mmu; } if (_MALI_OSK_ERR_OK == mali_mmu_reset(mmu)) { /* Setup IRQ handlers (which will do IRQ probing if needed) */ mmu->irq = _mali_osk_irq_init(resource->irq, mali_group_upper_half_mmu, group, mali_mmu_probe_trigger, mali_mmu_probe_ack, mmu, "mali_mmu_irq_handlers"); if (NULL != mmu->irq) { return mmu; } else { MALI_PRINT_ERROR(("Mali MMU: Failed to setup interrupt handlers for MMU %s\n", mmu->hw_core.description)); } } mali_group_remove_mmu_core(group); } else { MALI_PRINT_ERROR(("Mali MMU: Failed to add core %s to group\n", mmu->hw_core.description)); } mali_hw_core_delete(&mmu->hw_core); } _mali_osk_free(mmu); } else { MALI_PRINT_ERROR(("Failed to allocate memory for MMU\n")); } return NULL; }
static ump_descriptor_table *descriptor_table_alloc(int count) { ump_descriptor_table *table; table = _mali_osk_calloc(1, sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG) + (sizeof(void *) * count)); if (NULL != table) { table->usage = (u32 *)((u8 *)table + sizeof(ump_descriptor_table)); table->mappings = (void **)((u8 *)table + sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG)); } return table; }
struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource) { struct mali_mmu_core* mmu = NULL; MALI_DEBUG_ASSERT_POINTER(resource); MALI_DEBUG_PRINT(2, ("Mali MMU: Creating Mali MMU: %s\n", resource->description)); mmu = _mali_osk_calloc(1,sizeof(struct mali_mmu_core)); if (NULL != mmu) { #ifdef CONFIG_ARCH_MESON6 mmu->id = resource->mmu_id; MALI_DEBUG_PRINT(3, ("Mali MMU: mmu_id: %d\n", resource->mmu_id)); #endif if (_MALI_OSK_ERR_OK == mali_hw_core_create(&mmu->hw_core, resource, MALI_MMU_REGISTERS_SIZE)) { if (_MALI_OSK_ERR_OK == mali_mmu_reset(mmu)) { /* Setup IRQ handlers (which will do IRQ probing if needed) */ mmu->irq = _mali_osk_irq_init(resource->irq, mali_mmu_upper_half, mali_mmu_bottom_half, mali_mmu_probe_trigger, mali_mmu_probe_ack, mmu, "mali_mmu_irq_handlers"); if (NULL != mmu->irq) { return mmu; } else { MALI_PRINT_ERROR(("Failed to setup interrupt handlers for MMU %s\n", mmu->hw_core.description)); } } mali_hw_core_delete(&mmu->hw_core); } _mali_osk_free(mmu); } else { MALI_PRINT_ERROR(("Failed to allocate memory for MMU\n")); } return NULL; }
/** * Creates a sync fence tracker and a sync fence. Adds sync fence tracker to Timeline system and * returns sync fence. The sync fence will be signaled when the sync fence tracker is activated. * * @param timeline Timeline. * @param point Point on timeline. * @return Sync fence that will be signaled when tracker is activated. */ static struct sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) { struct mali_timeline_sync_fence_tracker *sync_fence_tracker; struct sync_fence *sync_fence; struct mali_timeline_fence fence; MALI_DEBUG_ASSERT_POINTER(timeline); MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); /* Allocate sync fence tracker. */ sync_fence_tracker = _mali_osk_calloc(1, sizeof(struct mali_timeline_sync_fence_tracker)); if (NULL == sync_fence_tracker) { MALI_PRINT_ERROR(("Mali Timeline: sync_fence_tracker allocation failed\n")); return NULL; } /* Create sync flag. */ MALI_DEBUG_ASSERT_POINTER(timeline->sync_tl); sync_fence_tracker->flag = mali_sync_flag_create(timeline->sync_tl, point); if (NULL == sync_fence_tracker->flag) { MALI_PRINT_ERROR(("Mali Timeline: sync_flag creation failed\n")); _mali_osk_free(sync_fence_tracker); return NULL; } /* Create sync fence from sync flag. */ sync_fence = mali_sync_flag_create_fence(sync_fence_tracker->flag); if (NULL == sync_fence) { MALI_PRINT_ERROR(("Mali Timeline: sync_fence creation failed\n")); mali_sync_flag_put(sync_fence_tracker->flag); _mali_osk_free(sync_fence_tracker); return NULL; } /* Setup fence for tracker. */ _mali_osk_memset(&fence, 0, sizeof(struct mali_timeline_fence)); fence.sync_fd = -1; fence.points[timeline->id] = point; /* Finally, add the tracker to Timeline system. */ mali_timeline_tracker_init(&sync_fence_tracker->tracker, MALI_TIMELINE_TRACKER_SYNC, &fence, sync_fence_tracker); point = mali_timeline_system_add_tracker(timeline->system, &sync_fence_tracker->tracker, MALI_TIMELINE_NONE); MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); return sync_fence; }
struct mali_spinlock_reentrant *mali_spinlock_reentrant_init(_mali_osk_lock_order_t lock_order) { struct mali_spinlock_reentrant *spinlock; spinlock = _mali_osk_calloc(1, sizeof(struct mali_spinlock_reentrant)); if (NULL == spinlock) { return NULL; } spinlock->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, lock_order); if (NULL == spinlock->lock) { mali_spinlock_reentrant_term(spinlock); return NULL; } return spinlock; }
struct mali_page_directory *mali_mmu_pagedir_alloc(void) { struct mali_page_directory *pagedir; pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory)); if(NULL == pagedir) { return NULL; } if(_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&pagedir->page_directory, &pagedir->page_directory_mapped)) { _mali_osk_free(pagedir); return NULL; } /* Zero page directory */ fill_page(pagedir->page_directory_mapped, 0); return pagedir; }
ump_descriptor_mapping *ump_descriptor_mapping_create(int init_entries, int max_entries) { ump_descriptor_mapping *map = _mali_osk_calloc(1, sizeof(ump_descriptor_mapping)); init_entries = MALI_PAD_INT(init_entries); max_entries = MALI_PAD_INT(max_entries); if (NULL != map) { map->table = descriptor_table_alloc(init_entries); if (NULL != map->table) { map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); if (NULL != map->lock) { _mali_osk_set_nonatomic_bit(0, map->table->usage); /* reserve bit 0 to prevent NULL/zero logic to kick in */ map->max_nr_mappings_allowed = max_entries; map->current_nr_mappings = init_entries; return map; } descriptor_table_free(map->table); } _mali_osk_free(map); } return NULL; }
struct mali_page_directory *mali_mmu_pagedir_alloc(void) { struct mali_page_directory *pagedir; _mali_osk_errcode_t err; mali_dma_addr phys; pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory)); if (NULL == pagedir) { return NULL; } err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped); if (_MALI_OSK_ERR_OK != err) { _mali_osk_free(pagedir); return NULL; } pagedir->page_directory = (u32)phys; /* Zero page directory */ fill_page(pagedir->page_directory_mapped, 0); return pagedir; }
_mali_osk_errcode_t mali_allocation_engine_allocate_memory(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor, mali_physical_memory_allocator * physical_allocators, _mali_osk_list_t *tracking_list ) { memory_engine * engine = (memory_engine*)mem_engine; MALI_DEBUG_ASSERT_POINTER(engine); MALI_DEBUG_ASSERT_POINTER(descriptor); MALI_DEBUG_ASSERT_POINTER(physical_allocators); /* ASSERT that the list member has been initialized, even if it won't be * used for tracking. We need it to be initialized to see if we need to * delete it from a list in the release function. */ MALI_DEBUG_ASSERT( NULL != descriptor->list.next && NULL != descriptor->list.prev ); if (_MALI_OSK_ERR_OK == engine->mali_address->allocate(descriptor)) { _mali_osk_errcode_t res = _MALI_OSK_ERR_OK; if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE ) { res = engine->process_address->allocate(descriptor); } if ( _MALI_OSK_ERR_OK == res ) { /* address space setup OK, commit physical memory to the allocation */ mali_physical_memory_allocator * active_allocator = physical_allocators; struct mali_physical_memory_allocation * active_allocation_tracker = &descriptor->physical_allocation; u32 offset = 0; while ( NULL != active_allocator ) { switch (active_allocator->allocate(active_allocator->ctx, mem_engine, descriptor, &offset, active_allocation_tracker)) { case MALI_MEM_ALLOC_FINISHED: if ( NULL != tracking_list ) { /* Insert into the memory session list */ /* ASSERT that it is not already part of a list */ MALI_DEBUG_ASSERT( _mali_osk_list_empty( &descriptor->list ) ); _mali_osk_list_add( &descriptor->list, tracking_list ); } MALI_SUCCESS; /* all done */ case MALI_MEM_ALLOC_NONE: /* reuse current active_allocation_tracker */ MALI_DEBUG_PRINT( 4, ("Memory Engine Allocate: No allocation on %s, resorting to %s\n", ( active_allocator->name ) ? active_allocator->name : "UNNAMED", ( active_allocator->next ) ? (( active_allocator->next->name )? active_allocator->next->name : "UNNAMED") : "NONE") ); active_allocator = active_allocator->next; break; case MALI_MEM_ALLOC_PARTIAL: if (NULL != active_allocator->next) { /* need a new allocation tracker */ active_allocation_tracker->next = _mali_osk_calloc(1, sizeof(mali_physical_memory_allocation)); if (NULL != active_allocation_tracker->next) { active_allocation_tracker = active_allocation_tracker->next; MALI_DEBUG_PRINT( 2, ("Memory Engine Allocate: Partial allocation on %s, resorting to %s\n", ( active_allocator->name ) ? active_allocator->name : "UNNAMED", ( active_allocator->next ) ? (( active_allocator->next->name )? active_allocator->next->name : "UNNAMED") : "NONE") ); active_allocator = active_allocator->next; break; } } /* FALL THROUGH */ case MALI_MEM_ALLOC_INTERNAL_FAILURE: active_allocator = NULL; /* end the while loop */ break; } } MALI_PRINT(("Memory allocate failed, could not allocate size %d kB.\n", descriptor->size/1024)); /* allocation failure, start cleanup */ /* loop over any potential partial allocations */ active_allocation_tracker = &descriptor->physical_allocation; while (NULL != active_allocation_tracker) { /* handle blank trackers which will show up during failure */ if (NULL != active_allocation_tracker->release) { active_allocation_tracker->release(active_allocation_tracker->ctx, active_allocation_tracker->handle); } active_allocation_tracker = active_allocation_tracker->next; } /* free the allocation tracker objects themselves, skipping the tracker stored inside the descriptor itself */ for ( active_allocation_tracker = descriptor->physical_allocation.next; active_allocation_tracker != NULL; ) { void * buf = active_allocation_tracker; active_allocation_tracker = active_allocation_tracker->next; _mali_osk_free(buf); } /* release the address spaces */ if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE ) { engine->process_address->release(descriptor); } } engine->mali_address->release(descriptor); } MALI_ERROR(_MALI_OSK_ERR_FAULT); }
/** * Allocate a fence waiter tracker. * * @return New fence waiter if successful, NULL if not. */ static struct mali_timeline_fence_wait_tracker *mali_timeline_fence_wait_tracker_alloc(void) { return (struct mali_timeline_fence_wait_tracker *) _mali_osk_calloc(1, sizeof(struct mali_timeline_fence_wait_tracker)); }
struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_uk_pp_start_job_s *uargs, u32 id) { struct mali_pp_job *job; u32 perf_counter_flag; job = _mali_osk_calloc(1, sizeof(struct mali_pp_job)); if (NULL != job) { if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) { goto fail; } if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) { MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n")); goto fail; } if (!mali_pp_job_use_no_notification(job)) { job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s)); if (NULL == job->finished_notification) goto fail; } perf_counter_flag = mali_pp_job_get_perf_counter_flag(job); /* case when no counters came from user space * so pass the debugfs / DS-5 provided global ones to the job object */ if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { mali_pp_job_set_perf_counter_src0(job, mali_pp_job_get_pp_counter_src0()); mali_pp_job_set_perf_counter_src1(job, mali_pp_job_get_pp_counter_src1()); } _mali_osk_list_init(&job->list); job->session = session; _mali_osk_list_init(&job->session_list); job->id = id; job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1; job->pid = _mali_osk_get_pid(); job->tid = _mali_osk_get_tid(); job->num_memory_cookies = job->uargs.num_memory_cookies; if (job->num_memory_cookies > 0) { u32 size; if (job->uargs.num_memory_cookies > session->descriptor_mapping->current_nr_mappings) { MALI_PRINT_ERROR(("Mali PP job: Too many memory cookies specified in job object\n")); goto fail; } size = sizeof(*job->uargs.memory_cookies) * job->num_memory_cookies; job->memory_cookies = _mali_osk_malloc(size); if (NULL == job->memory_cookies) { MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size)); goto fail; } if (0 != _mali_osk_copy_from_user(job->memory_cookies, job->uargs.memory_cookies, size)) { MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size)); goto fail; } #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) job->num_dma_bufs = job->num_memory_cookies; job->dma_bufs = _mali_osk_calloc(job->num_dma_bufs, sizeof(struct mali_dma_buf_attachment *)); if (NULL == job->dma_bufs) { MALI_PRINT_ERROR(("Mali PP job: Failed to allocate dma_bufs array!\n")); goto fail; } #endif } else { job->memory_cookies = NULL; } return job; } fail: if (NULL != job) { mali_pp_job_delete(job); } return NULL; }
_mali_osk_errcode_t _ump_ukk_allocate( _ump_uk_allocate_s *user_interaction ) { ump_session_data * session_data = NULL; ump_dd_mem *new_allocation = NULL; ump_session_memory_list_element * session_memory_element = NULL; int map_id; DEBUG_ASSERT_POINTER( user_interaction ); DEBUG_ASSERT_POINTER( user_interaction->ctx ); session_data = (ump_session_data *) user_interaction->ctx; session_memory_element = _mali_osk_calloc( 1, sizeof(ump_session_memory_list_element)); if (NULL == session_memory_element) { DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n")); return _MALI_OSK_ERR_NOMEM; } new_allocation = _mali_osk_calloc( 1, sizeof(ump_dd_mem)); if (NULL==new_allocation) { _mali_osk_free(session_memory_element); DBG_MSG(1, ("Failed to allocate ump_dd_mem in _ump_ukk_allocate()\n")); return _MALI_OSK_ERR_NOMEM; } /* Create a secure ID for this allocation */ _mali_osk_lock_wait(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW); map_id = ump_descriptor_mapping_allocate_mapping(device.secure_id_map, (void*)new_allocation); if (map_id < 0) { _mali_osk_lock_signal(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW); _mali_osk_free(session_memory_element); _mali_osk_free(new_allocation); DBG_MSG(1, ("Failed to allocate secure ID in ump_ioctl_allocate()\n")); return - _MALI_OSK_ERR_INVALID_FUNC; } /* Initialize the part of the new_allocation that we know so for */ new_allocation->secure_id = (ump_secure_id)map_id; _mali_osk_atomic_init(&new_allocation->ref_count,1); if ( 0==(UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE & user_interaction->constraints) ) new_allocation->is_cached = 0; else new_allocation->is_cached = 1; /* special case a size of 0, we should try to emulate what malloc does in this case, which is to return a valid pointer that must be freed, but can't be dereferences */ if (0 == user_interaction->size) { user_interaction->size = 1; /* emulate by actually allocating the minimum block size */ } new_allocation->size_bytes = UMP_SIZE_ALIGN(user_interaction->size); /* Page align the size */ new_allocation->lock_usage = UMP_NOT_LOCKED; /* Now, ask the active memory backend to do the actual memory allocation */ if (!device.backend->allocate( device.backend->ctx, new_allocation ) ) { DBG_MSG(3, ("OOM: No more UMP memory left. Failed to allocate memory in ump_ioctl_allocate(). Size: %lu, requested size: %lu\n", new_allocation->size_bytes, (unsigned long)user_interaction->size)); ump_descriptor_mapping_free(device.secure_id_map, map_id); _mali_osk_lock_signal(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW); _mali_osk_free(new_allocation); _mali_osk_free(session_memory_element); return _MALI_OSK_ERR_INVALID_FUNC; } new_allocation->hw_device = _UMP_UK_USED_BY_CPU; new_allocation->ctx = device.backend->ctx; new_allocation->release_func = device.backend->release; _mali_osk_lock_signal(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW); /* Initialize the session_memory_element, and add it to the session object */ session_memory_element->mem = new_allocation; _mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW); _mali_osk_list_add(&(session_memory_element->list), &(session_data->list_head_session_memory_list)); _mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW); user_interaction->secure_id = new_allocation->secure_id; user_interaction->size = new_allocation->size_bytes; DBG_MSG(3, ("UMP memory allocated. ID: %u, size: %lu\n", new_allocation->secure_id, new_allocation->size_bytes)); return _MALI_OSK_ERR_OK; }
int ump_dmabuf_import_wrapper(u32 __user *argument, struct ump_session_data *session_data) { ump_session_memory_list_element *session = NULL; struct ump_uk_dmabuf ump_dmabuf; ump_dd_handle *ump_handle; ump_dd_physical_block *blocks; struct dma_buf_attachment *attach; struct dma_buf *dma_buf; struct sg_table *sgt; struct scatterlist *sgl; unsigned long block_size; /* FIXME */ struct device dev; unsigned int i = 0, npages; int ret; /* Sanity check input parameters */ if (!argument || !session_data) { MSG_ERR(("NULL parameter.\n")); return -EINVAL; } if (copy_from_user(&ump_dmabuf, argument, sizeof(struct ump_uk_dmabuf))) { MSG_ERR(("copy_from_user() failed.\n")); return -EFAULT; } dma_buf = dma_buf_get(ump_dmabuf.fd); if (IS_ERR(dma_buf)) return PTR_ERR(dma_buf); /* * check whether dma_buf imported already exists or not. * * TODO * if already imported then dma_buf_put() should be called * and then just return dma_buf imported. */ attach = dma_buf_attach(dma_buf, &dev); if (IS_ERR(attach)) { ret = PTR_ERR(attach); goto err_dma_buf_put; } sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); if (IS_ERR(sgt)) { ret = PTR_ERR(sgt); goto err_dma_buf_detach; } npages = sgt->nents; /* really need? */ ump_dmabuf.ctx = (void *)session_data; block_size = sizeof(ump_dd_physical_block) * npages; blocks = (ump_dd_physical_block *)_mali_osk_malloc(block_size); if (NULL == blocks) { MSG_ERR(("Failed to allocate blocks\n")); ret = -ENOMEM; goto err_dmu_buf_unmap; } sgl = sgt->sgl; while (i < npages) { blocks[i].addr = sg_phys(sgl); blocks[i].size = sg_dma_len(sgl); sgl = sg_next(sgl); i++; } /* * Initialize the session memory list element, and add it * to the session object */ session = _mali_osk_calloc(1, sizeof(*session)); if (!session) { DBG_MSG(1, ("Failed to allocate session.\n")); ret = -EFAULT; goto err_free_block; } ump_handle = ump_dd_handle_create_from_phys_blocks(blocks, i); if (UMP_DD_HANDLE_INVALID == ump_handle) { DBG_MSG(1, ("Failed to create ump handle.\n")); ret = -EFAULT; goto err_free_session; } session->mem = (ump_dd_mem *)ump_handle; _mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW); _mali_osk_list_add(&(session->list), &(session_data->list_head_session_memory_list)); _mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW); _mali_osk_free(blocks); ump_dmabuf.ump_handle = (uint32_t)ump_handle; ump_dmabuf.size = ump_dd_size_get(ump_handle); if (copy_to_user(argument, &ump_dmabuf, sizeof(struct ump_uk_dmabuf))) { MSG_ERR(("copy_to_user() failed.\n")); ret = -EFAULT; goto err_release_ump_handle; } return 0; err_release_ump_handle: ump_dd_reference_release(ump_handle); err_free_session: _mali_osk_free(session); err_free_block: _mali_osk_free(blocks); err_dmu_buf_unmap: dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); err_dma_buf_detach: dma_buf_detach(dma_buf, attach); err_dma_buf_put: dma_buf_put(dma_buf); return ret; }
/* * IOCTL operation; Import fd to UMP memory */ int ump_ion_import_wrapper(u32 __user * argument, struct ump_session_data * session_data) { _ump_uk_ion_import_s user_interaction; ump_dd_handle *ump_handle; ump_dd_physical_block * blocks; unsigned long num_blocks; struct ion_handle *ion_hnd; struct scatterlist *sg; struct scatterlist *sg_ion; unsigned long i = 0; ump_session_memory_list_element * session_memory_element = NULL; if (ion_client_ump==NULL) ion_client_ump = ion_client_create(ion_exynos, -1, "ump"); /* Sanity check input parameters */ if (NULL == argument || NULL == session_data) { MSG_ERR(("NULL parameter in ump_ioctl_allocate()\n")); return -ENOTTY; } /* Copy the user space memory to kernel space (so we safely can read it) */ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { MSG_ERR(("copy_from_user() in ump_ioctl_allocate()\n")); return -EFAULT; } user_interaction.ctx = (void *) session_data; /* translate fd to secure ID*/ ion_hnd = ion_import_fd(ion_client_ump, user_interaction.ion_fd); sg_ion = ion_map_dma(ion_client_ump,ion_hnd); blocks = (ump_dd_physical_block*)_mali_osk_malloc(sizeof(ump_dd_physical_block)*1024); if (NULL == blocks) { MSG_ERR(("Failed to allocate blocks in ump_ioctl_allocate()\n")); return -ENOMEM; } sg = sg_ion; do { blocks[i].addr = sg_phys(sg); blocks[i].size = sg_dma_len(sg); i++; if (i>=1024) { _mali_osk_free(blocks); MSG_ERR(("ion_import fail() in ump_ioctl_allocate()\n")); return -EFAULT; } sg = sg_next(sg); } while(sg); num_blocks = i; /* Initialize the session_memory_element, and add it to the session object */ session_memory_element = _mali_osk_calloc( 1, sizeof(ump_session_memory_list_element)); if (NULL == session_memory_element) { _mali_osk_free(blocks); DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n")); return -EFAULT; } ump_handle = ump_dd_handle_create_from_phys_blocks(blocks, num_blocks); if (UMP_DD_HANDLE_INVALID == ump_handle) { _mali_osk_free(session_memory_element); _mali_osk_free(blocks); DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n")); return -EFAULT; } session_memory_element->mem = (ump_dd_mem*)ump_handle; _mali_osk_mutex_wait(session_data->lock); _mali_osk_list_add(&(session_memory_element->list), &(session_data->list_head_session_memory_list)); _mali_osk_mutex_signal(session_data->lock); ion_unmap_dma(ion_client_ump,ion_hnd); ion_free(ion_client_ump, ion_hnd); _mali_osk_free(blocks); user_interaction.secure_id = ump_dd_secure_id_get(ump_handle); user_interaction.size = ump_dd_size_get(ump_handle); user_interaction.ctx = NULL; if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { /* If the copy fails then we should release the memory. We can use the IOCTL release to accomplish this */ MSG_ERR(("copy_to_user() failed in ump_ioctl_allocate()\n")); return -EFAULT; } return 0; /* success */ }
struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_uk_pp_start_job_s __user *uargs, u32 id) { struct mali_pp_job *job; u32 perf_counter_flag; job = _mali_osk_calloc(1, sizeof(struct mali_pp_job)); if (NULL != job) { if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) { goto fail; } if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) { MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n")); goto fail; } if (!mali_pp_job_use_no_notification(job)) { job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s)); if (NULL == job->finished_notification) goto fail; } perf_counter_flag = mali_pp_job_get_perf_counter_flag(job); /* case when no counters came from user space * so pass the debugfs / DS-5 provided global ones to the job object */ if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count); /* These counters apply for all virtual jobs, and where no per sub job counter is specified */ job->uargs.perf_counter_src0 = pp_counter_src0; job->uargs.perf_counter_src1 = pp_counter_src1; /* We only copy the per sub job array if it is enabled with at least one counter */ if (0 < sub_job_count) { job->perf_counter_per_sub_job_count = sub_job_count; _mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0)); _mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1)); } } _mali_osk_list_init(&job->list); job->session = session; _mali_osk_list_init(&job->session_list); job->id = id; job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1; job->pid = _mali_osk_get_pid(); job->tid = _mali_osk_get_tid(); job->num_memory_cookies = job->uargs.num_memory_cookies; if (job->num_memory_cookies > 0) { u32 size; u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies; if (job->uargs.num_memory_cookies > session->descriptor_mapping->current_nr_mappings) { MALI_PRINT_ERROR(("Mali PP job: Too many memory cookies specified in job object\n")); goto fail; } size = sizeof(*memory_cookies) * job->num_memory_cookies; job->memory_cookies = _mali_osk_malloc(size); if (NULL == job->memory_cookies) { MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size)); goto fail; } if (0 != _mali_osk_copy_from_user(job->memory_cookies, memory_cookies, size)) { MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size)); goto fail; } #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) job->num_dma_bufs = job->num_memory_cookies; job->dma_bufs = _mali_osk_calloc(job->num_dma_bufs, sizeof(struct mali_dma_buf_attachment *)); if (NULL == job->dma_bufs) { MALI_PRINT_ERROR(("Mali PP job: Failed to allocate dma_bufs array!\n")); goto fail; } #endif } /* Prepare DMA command buffer to start job, if it is virtual. */ if (mali_pp_job_is_virtual_group_job(job)) { struct mali_pp_core *core; _mali_osk_errcode_t err = mali_dma_get_cmd_buf(&job->dma_cmd_buf); if (_MALI_OSK_ERR_OK != err) { MALI_PRINT_ERROR(("Mali PP job: Failed to allocate DMA command buffer\n")); goto fail; } core = mali_pp_scheduler_get_virtual_pp(); MALI_DEBUG_ASSERT_POINTER(core); mali_pp_job_dma_cmd_prepare(core, job, 0, &job->dma_cmd_buf); } if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) { /* Not a valid job. */ goto fail; } mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job); mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); return job; } fail: if (NULL != job) { mali_pp_job_delete(job); } return NULL; }
struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual, u32 bcast_id) { struct mali_pp_core *core = NULL; MALI_DEBUG_PRINT(2, ("Mali PP: Creating Mali PP core: %s\n", resource->description)); MALI_DEBUG_PRINT(2, ("Mali PP: Base address of PP core: 0x%x\n", resource->base)); if (mali_global_num_pp_cores >= MALI_MAX_NUMBER_OF_PP_CORES) { MALI_PRINT_ERROR(("Mali PP: Too many PP core objects created\n")); return NULL; } core = _mali_osk_calloc(1, sizeof(struct mali_pp_core)); if (NULL != core) { core->core_id = mali_global_num_pp_cores; core->bcast_id = bcast_id; if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALI200_REG_SIZEOF_REGISTER_BANK)) { _mali_osk_errcode_t ret; if (!is_virtual) { ret = mali_pp_reset(core); } else { ret = _MALI_OSK_ERR_OK; } if (_MALI_OSK_ERR_OK == ret) { ret = mali_group_add_pp_core(group, core); if (_MALI_OSK_ERR_OK == ret) { /* Setup IRQ handlers (which will do IRQ probing if needed) */ MALI_DEBUG_ASSERT(!is_virtual || -1 != resource->irq); core->irq = _mali_osk_irq_init(resource->irq, mali_group_upper_half_pp, group, mali_pp_irq_probe_trigger, mali_pp_irq_probe_ack, core, resource->description); if (NULL != core->irq) { mali_global_pp_cores[mali_global_num_pp_cores] = core; mali_global_num_pp_cores++; return core; } else { MALI_PRINT_ERROR(("Mali PP: Failed to setup interrupt handlers for PP core %s\n", core->hw_core.description)); } mali_group_remove_pp_core(group); } else { MALI_PRINT_ERROR(("Mali PP: Failed to add core %s to group\n", core->hw_core.description)); } } mali_hw_core_delete(&core->hw_core); } _mali_osk_free(core); } else { MALI_PRINT_ERROR(("Mali PP: Failed to allocate memory for PP core\n")); } return NULL; }
int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs) { _mali_uk_profiling_control_set_s kargs; _mali_osk_errcode_t err; u8 *kernel_control_data = NULL; u8 *kernel_response_data = NULL; MALI_CHECK_NON_NULL(uargs, -EINVAL); if (0 != get_user(kargs.control_packet_size, &uargs->control_packet_size)) return -EFAULT; if (0 != get_user(kargs.response_packet_size, &uargs->response_packet_size)) return -EFAULT; kargs.ctx = (uintptr_t)session_data; if (0 != kargs.control_packet_size) { kernel_control_data = _mali_osk_calloc(1, kargs.control_packet_size); if (NULL == kernel_control_data) { return -ENOMEM; } MALI_DEBUG_ASSERT(0 != kargs.response_packet_size); kernel_response_data = _mali_osk_calloc(1, kargs.response_packet_size); if (NULL == kernel_response_data) { _mali_osk_free(kernel_control_data); return -ENOMEM; } kargs.control_packet_data = (uintptr_t)kernel_control_data; kargs.response_packet_data = (uintptr_t)kernel_response_data; if (0 != copy_from_user((void *)(uintptr_t)kernel_control_data, (void *)(uintptr_t)uargs->control_packet_data, kargs.control_packet_size)) { _mali_osk_free(kernel_control_data); _mali_osk_free(kernel_response_data); return -EFAULT; } err = _mali_ukk_profiling_control_set(&kargs); if (_MALI_OSK_ERR_OK != err) { _mali_osk_free(kernel_control_data); _mali_osk_free(kernel_response_data); return map_errcode(err); } if (0 != kargs.response_packet_size && 0 != copy_to_user(((void *)(uintptr_t)uargs->response_packet_data), ((void *)(uintptr_t)kargs.response_packet_data), kargs.response_packet_size)) { _mali_osk_free(kernel_control_data); _mali_osk_free(kernel_response_data); return -EFAULT; } if (0 != put_user(kargs.response_packet_size, &uargs->response_packet_size)) { _mali_osk_free(kernel_control_data); _mali_osk_free(kernel_response_data); return -EFAULT; } _mali_osk_free(kernel_control_data); _mali_osk_free(kernel_response_data); } else { err = _mali_ukk_profiling_control_set(&kargs); if (_MALI_OSK_ERR_OK != err) { return map_errcode(err); } } return 0; }