static void free_buffer_page(struct ion_system_heap *heap,
struct ion_buffer *buffer, struct page *page,
unsigned int order)
{
bool cached = ion_buffer_cached(buffer);
bool split_pages = ion_buffer_fault_user_mappings(buffer);
int i;
if ((buffer->flags & ION_FLAG_FREED_FROM_SHRINKER)) {
ion_alloc_dec_usage(ION_TOTAL, 1 << order);
if (split_pages) {
for (i = 0; i < (1 << order); i++)
__free_page(page + i);
} else {
__free_pages(page, order);
}
} else {
struct ion_page_pool *pool;
if (cached)
pool = heap->cached_pools[order_to_index(order)];
else
pool = heap->uncached_pools[order_to_index(order)];
ion_page_pool_free(pool, page);
}
}
static void free_buffer_page(struct ion_system_heap *heap,
struct ion_buffer *buffer, struct page *page,
unsigned int order)
{
bool cached = ion_buffer_cached(buffer);
if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) {
struct ion_page_pool *pool = heap->pools[order_to_index(order)];
ion_page_pool_free(pool, page);
} else {
__free_pages(page, order);
}
}
static struct page_info *alloc_largest_available(struct ion_system_heap *heap,
struct ion_buffer *buffer,
unsigned long size,
unsigned int max_order)
{
struct page *page;
struct page_info *info;
int i;
struct ion_page_pool *pool;
bool from_pool = false;
info = kmalloc(sizeof(struct page_info), GFP_KERNEL);
if (!info)
return NULL;
for (i = 0; i < num_orders; i++) {
if (size < order_to_size(orders[i]))
continue;
if (max_order < orders[i])
continue;
if (!ion_buffer_cached(buffer)) {
pool = heap->pools[order_to_index(orders[i])];
mutex_lock(&pool->mutex);
if ((pool->high_count > 0) || (pool->low_count > 0))
from_pool = true;
mutex_unlock(&pool->mutex);
}
page = alloc_buffer_page(heap, buffer, orders[i]);
if (!page)
continue;
info->page = page;
info->order = orders[i];
info->from_pool = from_pool;
INIT_LIST_HEAD(&info->list);
return info;
}
kfree(info);
return NULL;
}
static struct page *alloc_buffer_page(struct ion_system_heap *heap,
struct ion_buffer *buffer,
unsigned long order,
bool *from_pool)
{
bool cached = ion_buffer_cached(buffer);
bool split_pages = ion_buffer_fault_user_mappings(buffer);
struct page *page;
struct ion_page_pool *pool;
if (!cached)
pool = heap->uncached_pools[order_to_index(order)];
else
pool = heap->cached_pools[order_to_index(order)];
page = ion_page_pool_alloc(pool, from_pool);
if (!page)
return 0;
if (split_pages)
split_page(page, order);
return page;
}
static struct page *alloc_buffer_page(struct ion_system_heap *heap,
struct ion_buffer *buffer,
unsigned long order)
{
bool cached = ion_buffer_cached(buffer);
struct ion_page_pool *pool = heap->pools[order_to_index(order)];
struct page *page;
if (!cached) {
page = ion_page_pool_alloc(pool);
} else {
gfp_t gfp_flags = low_order_gfp_flags;
if (order > 4)
gfp_flags = high_order_gfp_flags;
page = alloc_pages(gfp_flags, order);
}
if (!page)
return NULL;
return page;
}
/* ION CMA heap operations functions */
static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer,
unsigned long len, unsigned long align,
unsigned long flags)
{
struct ion_cma_heap *cma_heap = to_cma_heap(heap);
struct device *dev = cma_heap->dev;
struct ion_cma_buffer_info *info;
dev_dbg(dev, "Request buffer allocation len %ld\n", len);
if (buffer->flags & ION_FLAG_CACHED)
return -EINVAL;
if (align > PAGE_SIZE)
return -EINVAL;
if (!ion_is_heap_available(heap, flags, NULL))
return -EPERM;
info = kzalloc(sizeof(struct ion_cma_buffer_info), GFP_KERNEL);
if (!info) {
dev_err(dev, "Can't allocate buffer info\n");
return ION_CMA_ALLOCATE_FAILED;
}
info->cpu_addr = dma_alloc_coherent(dev, len, &(info->handle),
GFP_HIGHUSER | __GFP_ZERO);
if (!info->cpu_addr) {
dev_err(dev, "Fail to allocate buffer\n");
goto err;
}
info->table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
if (!info->table) {
dev_err(dev, "Fail to allocate sg table\n");
goto free_mem;
}
if (ion_cma_get_sgtable
(dev, info->table, info->cpu_addr, info->handle, len))
goto free_table;
/* keep this for memory release */
buffer->priv_virt = info;
#ifdef CONFIG_ARM64
if (!ion_buffer_cached(buffer) && !(buffer->flags & ION_FLAG_PROTECTED)) {
if (ion_buffer_need_flush_all(buffer))
flush_all_cpu_caches();
else
__flush_dcache_area(page_address(sg_page(info->table->sgl)),
len);
}
#else
if (!ion_buffer_cached(buffer) && !(buffer->flags & ION_FLAG_PROTECTED)) {
if (ion_buffer_need_flush_all(buffer))
flush_all_cpu_caches();
else
dmac_flush_range(page_address(sg_page(info->table->sgl),
len, DMA_BIDIRECTIONAL));
}
#endif
if ((buffer->flags & ION_FLAG_PROTECTED) && ion_secure_protect(heap))
goto free_table;
dev_dbg(dev, "Allocate buffer %p\n", buffer);
return 0;
free_table:
kfree(info->table);
free_mem:
dma_free_coherent(dev, len, info->cpu_addr, info->handle);
err:
kfree(info);
return ION_CMA_ALLOCATE_FAILED;
}
