void mali_session_add(struct mali_session_data *session)
{
mali_session_lock();
_mali_osk_list_add(&session->link, &mali_sessions);
mali_session_count++;
mali_session_unlock();
}
struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session)
{
u32 i;
struct mali_soft_job_system *system;
struct mali_soft_job *job;
MALI_DEBUG_ASSERT_POINTER(session);
system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system));
if (NULL == system) {
return NULL;
}
system->session = session;
system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER);
if (NULL == system->lock) {
mali_soft_job_system_destroy(system);
return NULL;
}
system->lock_owner = 0;
_MALI_OSK_INIT_LIST_HEAD(&(system->jobs_free));
_MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used));
for (i = 0; i < MALI_MAX_NUM_SOFT_JOBS; ++i) {
job = &(system->jobs[i]);
_mali_osk_list_add(&(job->system_list), &(system->jobs_free));
job->system = system;
job->state = MALI_SOFT_JOB_STATE_FREE;
job->id = i;
}
return system;
}
struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job)
{
struct mali_soft_job *job;
_mali_osk_notification_t *notification = NULL;
MALI_DEBUG_ASSERT_POINTER(system);
MALI_DEBUG_ASSERT((MALI_SOFT_JOB_TYPE_USER_SIGNALED == type) ||
(MALI_SOFT_JOB_TYPE_SELF_SIGNALED == type));
notification = _mali_osk_notification_create(_MALI_NOTIFICATION_SOFT_ACTIVATED, sizeof(_mali_uk_soft_job_activated_s));
if (unlikely(NULL == notification)) {
MALI_PRINT_ERROR(("Mali Soft Job: failed to allocate notification"));
return NULL;
}
job = _mali_osk_malloc(sizeof(struct mali_soft_job));
if (unlikely(NULL == job)) {
MALI_DEBUG_PRINT(2, ("Mali Soft Job: system alloc job failed. \n"));
return NULL;
}
mali_soft_job_system_lock(system);
job->system = system;
job->id = system->last_job_id++;
job->state = MALI_SOFT_JOB_STATE_ALLOCATED;
_mali_osk_list_add(&(job->system_list), &(system->jobs_used));
job->type = type;
job->user_job = user_job;
job->activated = MALI_FALSE;
job->activated_notification = notification;
_mali_osk_atomic_init(&job->refcount, 1);
MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state);
MALI_DEBUG_ASSERT(system == job->system);
MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id);
mali_soft_job_system_unlock(system);
return job;
}
/*
* IOCTL operation; Import fd to UMP memory
*/
int ump_ion_import_wrapper(u32 __user * argument, struct ump_session_data * session_data)
{
_ump_uk_ion_import_s user_interaction;
ump_dd_handle *ump_handle;
ump_dd_physical_block * blocks;
unsigned long num_blocks;
struct ion_handle *ion_hnd;
struct scatterlist *sg;
struct scatterlist *sg_ion;
unsigned long i = 0;
ump_session_memory_list_element * session_memory_element = NULL;
if (ion_client_ump==NULL)
ion_client_ump = ion_client_create(ion_exynos, -1, "ump");
/* Sanity check input parameters */
if (NULL == argument || NULL == session_data)
{
MSG_ERR(("NULL parameter in ump_ioctl_allocate()\n"));
return -ENOTTY;
}
/* Copy the user space memory to kernel space (so we safely can read it) */
if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction)))
{
MSG_ERR(("copy_from_user() in ump_ioctl_allocate()\n"));
return -EFAULT;
}
user_interaction.ctx = (void *) session_data;
/* translate fd to secure ID*/
ion_hnd = ion_import_fd(ion_client_ump, user_interaction.ion_fd);
sg_ion = ion_map_dma(ion_client_ump,ion_hnd);
blocks = (ump_dd_physical_block*)_mali_osk_malloc(sizeof(ump_dd_physical_block)*1024);
if (NULL == blocks) {
MSG_ERR(("Failed to allocate blocks in ump_ioctl_allocate()\n"));
return -ENOMEM;
}
sg = sg_ion;
do {
blocks[i].addr = sg_phys(sg);
blocks[i].size = sg_dma_len(sg);
i++;
if (i>=1024) {
_mali_osk_free(blocks);
MSG_ERR(("ion_import fail() in ump_ioctl_allocate()\n"));
return -EFAULT;
}
sg = sg_next(sg);
} while(sg);
num_blocks = i;
/* Initialize the session_memory_element, and add it to the session object */
session_memory_element = _mali_osk_calloc( 1, sizeof(ump_session_memory_list_element));
if (NULL == session_memory_element)
{
_mali_osk_free(blocks);
DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n"));
return -EFAULT;
}
ump_handle = ump_dd_handle_create_from_phys_blocks(blocks, num_blocks);
if (UMP_DD_HANDLE_INVALID == ump_handle)
{
_mali_osk_free(session_memory_element);
_mali_osk_free(blocks);
DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n"));
return -EFAULT;
}
session_memory_element->mem = (ump_dd_mem*)ump_handle;
_mali_osk_mutex_wait(session_data->lock);
_mali_osk_list_add(&(session_memory_element->list), &(session_data->list_head_session_memory_list));
_mali_osk_mutex_signal(session_data->lock);
ion_unmap_dma(ion_client_ump,ion_hnd);
ion_free(ion_client_ump, ion_hnd);
_mali_osk_free(blocks);
user_interaction.secure_id = ump_dd_secure_id_get(ump_handle);
user_interaction.size = ump_dd_size_get(ump_handle);
user_interaction.ctx = NULL;
if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction)))
{
/* If the copy fails then we should release the memory. We can use the IOCTL release to accomplish this */
MSG_ERR(("copy_to_user() failed in ump_ioctl_allocate()\n"));
return -EFAULT;
}
return 0; /* success */
}
_mali_osk_errcode_t _ump_ukk_allocate( _ump_uk_allocate_s *user_interaction )
{
ump_session_data * session_data = NULL;
ump_dd_mem *new_allocation = NULL;
ump_session_memory_list_element * session_memory_element = NULL;
int map_id;
DEBUG_ASSERT_POINTER( user_interaction );
DEBUG_ASSERT_POINTER( user_interaction->ctx );
session_data = (ump_session_data *) user_interaction->ctx;
session_memory_element = _mali_osk_calloc( 1, sizeof(ump_session_memory_list_element));
if (NULL == session_memory_element)
{
DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n"));
return _MALI_OSK_ERR_NOMEM;
}
new_allocation = _mali_osk_calloc( 1, sizeof(ump_dd_mem));
if (NULL==new_allocation)
{
_mali_osk_free(session_memory_element);
DBG_MSG(1, ("Failed to allocate ump_dd_mem in _ump_ukk_allocate()\n"));
return _MALI_OSK_ERR_NOMEM;
}
/* Create a secure ID for this allocation */
_mali_osk_lock_wait(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW);
map_id = ump_descriptor_mapping_allocate_mapping(device.secure_id_map, (void*)new_allocation);
if (map_id < 0)
{
_mali_osk_lock_signal(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW);
_mali_osk_free(session_memory_element);
_mali_osk_free(new_allocation);
DBG_MSG(1, ("Failed to allocate secure ID in ump_ioctl_allocate()\n"));
return - _MALI_OSK_ERR_INVALID_FUNC;
}
/* Initialize the part of the new_allocation that we know so for */
new_allocation->secure_id = (ump_secure_id)map_id;
_mali_osk_atomic_init(&new_allocation->ref_count,1);
if ( 0==(UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE & user_interaction->constraints) )
new_allocation->is_cached = 0;
else new_allocation->is_cached = 1;
/* special case a size of 0, we should try to emulate what malloc does in this case, which is to return a valid pointer that must be freed, but can't be dereferences */
if (0 == user_interaction->size)
{
user_interaction->size = 1; /* emulate by actually allocating the minimum block size */
}
new_allocation->size_bytes = UMP_SIZE_ALIGN(user_interaction->size); /* Page align the size */
new_allocation->lock_usage = UMP_NOT_LOCKED;
/* Now, ask the active memory backend to do the actual memory allocation */
if (!device.backend->allocate( device.backend->ctx, new_allocation ) )
{
DBG_MSG(3, ("OOM: No more UMP memory left. Failed to allocate memory in ump_ioctl_allocate(). Size: %lu, requested size: %lu\n", new_allocation->size_bytes, (unsigned long)user_interaction->size));
ump_descriptor_mapping_free(device.secure_id_map, map_id);
_mali_osk_lock_signal(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW);
_mali_osk_free(new_allocation);
_mali_osk_free(session_memory_element);
return _MALI_OSK_ERR_INVALID_FUNC;
}
new_allocation->hw_device = _UMP_UK_USED_BY_CPU;
new_allocation->ctx = device.backend->ctx;
new_allocation->release_func = device.backend->release;
_mali_osk_lock_signal(device.secure_id_map_lock, _MALI_OSK_LOCKMODE_RW);
/* Initialize the session_memory_element, and add it to the session object */
session_memory_element->mem = new_allocation;
_mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW);
_mali_osk_list_add(&(session_memory_element->list), &(session_data->list_head_session_memory_list));
_mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW);
user_interaction->secure_id = new_allocation->secure_id;
user_interaction->size = new_allocation->size_bytes;
DBG_MSG(3, ("UMP memory allocated. ID: %u, size: %lu\n", new_allocation->secure_id, new_allocation->size_bytes));
return _MALI_OSK_ERR_OK;
}
_mali_osk_errcode_t mali_allocation_engine_allocate_memory(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor, mali_physical_memory_allocator * physical_allocators, _mali_osk_list_t *tracking_list )
{
memory_engine * engine = (memory_engine*)mem_engine;
MALI_DEBUG_ASSERT_POINTER(engine);
MALI_DEBUG_ASSERT_POINTER(descriptor);
MALI_DEBUG_ASSERT_POINTER(physical_allocators);
/* ASSERT that the list member has been initialized, even if it won't be
* used for tracking. We need it to be initialized to see if we need to
* delete it from a list in the release function. */
MALI_DEBUG_ASSERT( NULL != descriptor->list.next && NULL != descriptor->list.prev );
if (_MALI_OSK_ERR_OK == engine->mali_address->allocate(descriptor))
{
_mali_osk_errcode_t res = _MALI_OSK_ERR_OK;
if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
{
res = engine->process_address->allocate(descriptor);
}
if ( _MALI_OSK_ERR_OK == res )
{
/* address space setup OK, commit physical memory to the allocation */
mali_physical_memory_allocator * active_allocator = physical_allocators;
struct mali_physical_memory_allocation * active_allocation_tracker = &descriptor->physical_allocation;
u32 offset = 0;
while ( NULL != active_allocator )
{
switch (active_allocator->allocate(active_allocator->ctx, mem_engine, descriptor, &offset, active_allocation_tracker))
{
case MALI_MEM_ALLOC_FINISHED:
if ( NULL != tracking_list )
{
/* Insert into the memory session list */
/* ASSERT that it is not already part of a list */
MALI_DEBUG_ASSERT( _mali_osk_list_empty( &descriptor->list ) );
_mali_osk_list_add( &descriptor->list, tracking_list );
}
MALI_SUCCESS; /* all done */
case MALI_MEM_ALLOC_NONE:
/* reuse current active_allocation_tracker */
MALI_DEBUG_PRINT( 4, ("Memory Engine Allocate: No allocation on %s, resorting to %s\n",
( active_allocator->name ) ? active_allocator->name : "UNNAMED",
( active_allocator->next ) ? (( active_allocator->next->name )? active_allocator->next->name : "UNNAMED") : "NONE") );
active_allocator = active_allocator->next;
break;
case MALI_MEM_ALLOC_PARTIAL:
if (NULL != active_allocator->next)
{
/* need a new allocation tracker */
active_allocation_tracker->next = _mali_osk_calloc(1, sizeof(mali_physical_memory_allocation));
if (NULL != active_allocation_tracker->next)
{
active_allocation_tracker = active_allocation_tracker->next;
MALI_DEBUG_PRINT( 2, ("Memory Engine Allocate: Partial allocation on %s, resorting to %s\n",
( active_allocator->name ) ? active_allocator->name : "UNNAMED",
( active_allocator->next ) ? (( active_allocator->next->name )? active_allocator->next->name : "UNNAMED") : "NONE") );
active_allocator = active_allocator->next;
break;
}
}
/* FALL THROUGH */
case MALI_MEM_ALLOC_INTERNAL_FAILURE:
active_allocator = NULL; /* end the while loop */
break;
}
}
MALI_PRINT(("Memory allocate failed, could not allocate size %d kB.\n", descriptor->size/1024));
/* allocation failure, start cleanup */
/* loop over any potential partial allocations */
active_allocation_tracker = &descriptor->physical_allocation;
while (NULL != active_allocation_tracker)
{
/* handle blank trackers which will show up during failure */
if (NULL != active_allocation_tracker->release)
{
active_allocation_tracker->release(active_allocation_tracker->ctx, active_allocation_tracker->handle);
}
active_allocation_tracker = active_allocation_tracker->next;
}
/* free the allocation tracker objects themselves, skipping the tracker stored inside the descriptor itself */
for ( active_allocation_tracker = descriptor->physical_allocation.next; active_allocation_tracker != NULL; )
{
void * buf = active_allocation_tracker;
active_allocation_tracker = active_allocation_tracker->next;
_mali_osk_free(buf);
}
/* release the address spaces */
if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
{
engine->process_address->release(descriptor);
}
}
engine->mali_address->release(descriptor);
}
MALI_ERROR(_MALI_OSK_ERR_FAULT);
}
int ump_dmabuf_import_wrapper(u32 __user *argument,
struct ump_session_data *session_data)
{
ump_session_memory_list_element *session = NULL;
struct ump_uk_dmabuf ump_dmabuf;
ump_dd_handle *ump_handle;
ump_dd_physical_block *blocks;
struct dma_buf_attachment *attach;
struct dma_buf *dma_buf;
struct sg_table *sgt;
struct scatterlist *sgl;
unsigned long block_size;
/* FIXME */
struct device dev;
unsigned int i = 0, npages;
int ret;
/* Sanity check input parameters */
if (!argument || !session_data) {
MSG_ERR(("NULL parameter.\n"));
return -EINVAL;
}
if (copy_from_user(&ump_dmabuf, argument,
sizeof(struct ump_uk_dmabuf))) {
MSG_ERR(("copy_from_user() failed.\n"));
return -EFAULT;
}
dma_buf = dma_buf_get(ump_dmabuf.fd);
if (IS_ERR(dma_buf))
return PTR_ERR(dma_buf);
/*
* check whether dma_buf imported already exists or not.
*
* TODO
* if already imported then dma_buf_put() should be called
* and then just return dma_buf imported.
*/
attach = dma_buf_attach(dma_buf, &dev);
if (IS_ERR(attach)) {
ret = PTR_ERR(attach);
goto err_dma_buf_put;
}
sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
if (IS_ERR(sgt)) {
ret = PTR_ERR(sgt);
goto err_dma_buf_detach;
}
npages = sgt->nents;
/* really need? */
ump_dmabuf.ctx = (void *)session_data;
block_size = sizeof(ump_dd_physical_block) * npages;
blocks = (ump_dd_physical_block *)_mali_osk_malloc(block_size);
if (NULL == blocks) {
MSG_ERR(("Failed to allocate blocks\n"));
ret = -ENOMEM;
goto err_dmu_buf_unmap;
}
sgl = sgt->sgl;
while (i < npages) {
blocks[i].addr = sg_phys(sgl);
blocks[i].size = sg_dma_len(sgl);
sgl = sg_next(sgl);
i++;
}
/*
* Initialize the session memory list element, and add it
* to the session object
*/
session = _mali_osk_calloc(1, sizeof(*session));
if (!session) {
DBG_MSG(1, ("Failed to allocate session.\n"));
ret = -EFAULT;
goto err_free_block;
}
ump_handle = ump_dd_handle_create_from_phys_blocks(blocks, i);
if (UMP_DD_HANDLE_INVALID == ump_handle) {
DBG_MSG(1, ("Failed to create ump handle.\n"));
ret = -EFAULT;
goto err_free_session;
}
session->mem = (ump_dd_mem *)ump_handle;
_mali_osk_lock_wait(session_data->lock, _MALI_OSK_LOCKMODE_RW);
_mali_osk_list_add(&(session->list),
&(session_data->list_head_session_memory_list));
_mali_osk_lock_signal(session_data->lock, _MALI_OSK_LOCKMODE_RW);
_mali_osk_free(blocks);
ump_dmabuf.ump_handle = (uint32_t)ump_handle;
ump_dmabuf.size = ump_dd_size_get(ump_handle);
if (copy_to_user(argument, &ump_dmabuf,
sizeof(struct ump_uk_dmabuf))) {
MSG_ERR(("copy_to_user() failed.\n"));
ret = -EFAULT;
goto err_release_ump_handle;
}
return 0;
err_release_ump_handle:
ump_dd_reference_release(ump_handle);
err_free_session:
_mali_osk_free(session);
err_free_block:
_mali_osk_free(blocks);
err_dmu_buf_unmap:
dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
err_dma_buf_detach:
dma_buf_detach(dma_buf, attach);
err_dma_buf_put:
dma_buf_put(dma_buf);
return ret;
}
_mali_osk_errcode_t mali_pp_scheduler_initialize(void)
{
struct mali_group *group;
struct mali_pp_core *pp_core;
_mali_osk_lock_flags_t lock_flags;
u32 num_groups;
u32 i;
#if defined(MALI_UPPER_HALF_SCHEDULING)
lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK_IRQ | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
#else
lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
#endif
_MALI_OSK_INIT_LIST_HEAD(&job_queue);
_MALI_OSK_INIT_LIST_HEAD(&group_list_working);
_MALI_OSK_INIT_LIST_HEAD(&group_list_idle);
_MALI_OSK_INIT_LIST_HEAD(&virtual_job_queue);
pp_scheduler_lock = _mali_osk_lock_init(lock_flags, 0, _MALI_OSK_LOCK_ORDER_SCHEDULER);
if (NULL == pp_scheduler_lock)
{
return _MALI_OSK_ERR_NOMEM;
}
pp_scheduler_working_wait_queue = _mali_osk_wait_queue_init();
if (NULL == pp_scheduler_working_wait_queue)
{
_mali_osk_lock_term(pp_scheduler_lock);
return _MALI_OSK_ERR_NOMEM;
}
pp_scheduler_wq_schedule = _mali_osk_wq_create_work(mali_pp_scheduler_do_schedule, NULL);
if (NULL == pp_scheduler_wq_schedule)
{
_mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
_mali_osk_lock_term(pp_scheduler_lock);
return _MALI_OSK_ERR_NOMEM;
}
#if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
pp_scheduler_wq_job_delete = _mali_osk_wq_create_work(mali_pp_scheduler_do_job_delete, NULL);
if (NULL == pp_scheduler_wq_job_delete)
{
_mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
_mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
_mali_osk_lock_term(pp_scheduler_lock);
return _MALI_OSK_ERR_NOMEM;
}
pp_scheduler_job_delete_lock = _mali_osk_lock_init(_MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK_IRQ |_MALI_OSK_LOCKFLAG_NONINTERRUPTABLE, 0, _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
if (NULL == pp_scheduler_job_delete_lock)
{
_mali_osk_wq_delete_work(pp_scheduler_wq_job_delete);
_mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
_mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
_mali_osk_lock_term(pp_scheduler_lock);
return _MALI_OSK_ERR_NOMEM;
}
#endif
num_groups = mali_group_get_glob_num_groups();
/* Do we have a virtual group? */
for (i = 0; i < num_groups; i++)
{
group = mali_group_get_glob_group(i);
if (mali_group_is_virtual(group))
{
MALI_DEBUG_PRINT(3, ("Found virtual group %p\n", group));
virtual_group = group;
break;
}
}
/* Find all the available PP cores */
for (i = 0; i < num_groups; i++)
{
group = mali_group_get_glob_group(i);
pp_core = mali_group_get_pp_core(group);
if (NULL != pp_core && !mali_group_is_virtual(group))
{
if (0 == pp_version)
{
/* Retrieve PP version from the first available PP core */
pp_version = mali_pp_core_get_version(pp_core);
}
if (NULL != virtual_group)
{
/* Add all physical PP cores to the virtual group */
mali_group_lock(virtual_group);
group->state = MALI_GROUP_STATE_JOINING_VIRTUAL;
mali_group_add_group(virtual_group, group);
mali_group_unlock(virtual_group);
}
else
{
_mali_osk_list_add(&group->pp_scheduler_list, &group_list_idle);
}
num_cores++;
}
}
return _MALI_OSK_ERR_OK;
}
