static void free_buffer_page(struct ion_system_heap *heap, struct ion_buffer *buffer, struct page *page, unsigned int order) { bool cached = ion_buffer_cached(buffer); bool split_pages = ion_buffer_fault_user_mappings(buffer); int i; if ((buffer->flags & ION_FLAG_FREED_FROM_SHRINKER)) { ion_alloc_dec_usage(ION_TOTAL, 1 << order); if (split_pages) { for (i = 0; i < (1 << order); i++) __free_page(page + i); } else { __free_pages(page, order); } } else { struct ion_page_pool *pool; if (cached) pool = heap->cached_pools[order_to_index(order)]; else pool = heap->uncached_pools[order_to_index(order)]; ion_page_pool_free(pool, page); } }
static void free_buffer_page(struct ion_system_heap *heap, struct ion_buffer *buffer, struct page *page, unsigned int order) { bool cached = ion_buffer_cached(buffer); if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) { struct ion_page_pool *pool = heap->pools[order_to_index(order)]; ion_page_pool_free(pool, page); } else { __free_pages(page, order); } }
static struct page_info *alloc_largest_available(struct ion_system_heap *heap, struct ion_buffer *buffer, unsigned long size, unsigned int max_order) { struct page *page; struct page_info *info; int i; struct ion_page_pool *pool; bool from_pool = false; info = kmalloc(sizeof(struct page_info), GFP_KERNEL); if (!info) return NULL; for (i = 0; i < num_orders; i++) { if (size < order_to_size(orders[i])) continue; if (max_order < orders[i]) continue; if (!ion_buffer_cached(buffer)) { pool = heap->pools[order_to_index(orders[i])]; mutex_lock(&pool->mutex); if ((pool->high_count > 0) || (pool->low_count > 0)) from_pool = true; mutex_unlock(&pool->mutex); } page = alloc_buffer_page(heap, buffer, orders[i]); if (!page) continue; info->page = page; info->order = orders[i]; info->from_pool = from_pool; INIT_LIST_HEAD(&info->list); return info; } kfree(info); return NULL; }
static struct page *alloc_buffer_page(struct ion_system_heap *heap, struct ion_buffer *buffer, unsigned long order, bool *from_pool) { bool cached = ion_buffer_cached(buffer); bool split_pages = ion_buffer_fault_user_mappings(buffer); struct page *page; struct ion_page_pool *pool; if (!cached) pool = heap->uncached_pools[order_to_index(order)]; else pool = heap->cached_pools[order_to_index(order)]; page = ion_page_pool_alloc(pool, from_pool); if (!page) return 0; if (split_pages) split_page(page, order); return page; }
static struct page *alloc_buffer_page(struct ion_system_heap *heap, struct ion_buffer *buffer, unsigned long order) { bool cached = ion_buffer_cached(buffer); struct ion_page_pool *pool = heap->pools[order_to_index(order)]; struct page *page; if (!cached) { page = ion_page_pool_alloc(pool); } else { gfp_t gfp_flags = low_order_gfp_flags; if (order > 4) gfp_flags = high_order_gfp_flags; page = alloc_pages(gfp_flags, order); } if (!page) return NULL; return page; }
/* ION CMA heap operations functions */ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, unsigned long len, unsigned long align, unsigned long flags) { struct ion_cma_heap *cma_heap = to_cma_heap(heap); struct device *dev = cma_heap->dev; struct ion_cma_buffer_info *info; dev_dbg(dev, "Request buffer allocation len %ld\n", len); if (buffer->flags & ION_FLAG_CACHED) return -EINVAL; if (align > PAGE_SIZE) return -EINVAL; if (!ion_is_heap_available(heap, flags, NULL)) return -EPERM; info = kzalloc(sizeof(struct ion_cma_buffer_info), GFP_KERNEL); if (!info) { dev_err(dev, "Can't allocate buffer info\n"); return ION_CMA_ALLOCATE_FAILED; } info->cpu_addr = dma_alloc_coherent(dev, len, &(info->handle), GFP_HIGHUSER | __GFP_ZERO); if (!info->cpu_addr) { dev_err(dev, "Fail to allocate buffer\n"); goto err; } info->table = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (!info->table) { dev_err(dev, "Fail to allocate sg table\n"); goto free_mem; } if (ion_cma_get_sgtable (dev, info->table, info->cpu_addr, info->handle, len)) goto free_table; /* keep this for memory release */ buffer->priv_virt = info; #ifdef CONFIG_ARM64 if (!ion_buffer_cached(buffer) && !(buffer->flags & ION_FLAG_PROTECTED)) { if (ion_buffer_need_flush_all(buffer)) flush_all_cpu_caches(); else __flush_dcache_area(page_address(sg_page(info->table->sgl)), len); } #else if (!ion_buffer_cached(buffer) && !(buffer->flags & ION_FLAG_PROTECTED)) { if (ion_buffer_need_flush_all(buffer)) flush_all_cpu_caches(); else dmac_flush_range(page_address(sg_page(info->table->sgl), len, DMA_BIDIRECTIONAL)); } #endif if ((buffer->flags & ION_FLAG_PROTECTED) && ion_secure_protect(heap)) goto free_table; dev_dbg(dev, "Allocate buffer %p\n", buffer); return 0; free_table: kfree(info->table); free_mem: dma_free_coherent(dev, len, info->cpu_addr, info->handle); err: kfree(info); return ION_CMA_ALLOCATE_FAILED; }