/**
* dma_release_from_contiguous() - release allocated pages
* @dev: Pointer to device for which the pages were allocated.
* @pages: Allocated pages.
* @count: Number of allocated pages.
*
* This function releases memory allocated by dma_alloc_from_contiguous().
* It returns false when provided pages do not belong to contiguous area and
* true otherwise.
*/
bool dma_release_from_contiguous(struct device *dev, struct page *pages,
int count)
{
struct cma *cma = dev_get_cma_area(dev);
unsigned long pfn;
if (!cma || !pages)
return false;
pr_debug("%s(page %p)\n", __func__, (void *)pages);
pfn = page_to_pfn(pages);
if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count)
return false;
VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);
__dma_remap(pages, count << PAGE_SHIFT, PAGE_KERNEL_EXEC);
mutex_lock(&cma_mutex);
bitmap_clear(cma->bitmap, pfn - cma->base_pfn, count);
free_contig_range(pfn, count);
mutex_unlock(&cma_mutex);
return true;
}
void *cma_map_kernel(u32 phys_addr, size_t size)
{
pgprot_t prot = __get_dma_pgprot(NULL, pgprot_kernel);
struct page *page = phys_to_page(phys_addr);
void *ptr = NULL;
if (unlikely(phys_addr < mem_start || phys_addr > mem_start + mem_size)) {
pr_err("%s(%d) err: phys_addr 0x%x invalid!\n", __func__, __LINE__, phys_addr);
return NULL;
}
// BUG_ON(unlikely(!pfn_valid(__phys_to_pfn(phys_addr))));
size = PAGE_ALIGN(size);
if (PageHighMem(page)) {
ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, __builtin_return_address(0));
if (!ptr) {
pr_err("%s(%d) err: __dma_alloc_remap failed!\n", __func__, __LINE__);
return NULL;
}
} else {
__dma_remap(page, size, prot);
ptr = page_address(page);
}
return ptr;
}
void cma_unmap_kernel(u32 phys_addr, size_t size, void *cpu_addr)
{
struct page *page = phys_to_page(phys_addr);
BUG_ON(unlikely(!pfn_valid(__phys_to_pfn(phys_addr))));
size = PAGE_ALIGN(size);
if (PageHighMem(page))
__dma_free_remap(cpu_addr, size);
else
__dma_remap(page, size, pgprot_kernel);
}
static void *__dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
struct dma_attrs *attrs)
{
if (dev == NULL) {
WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
return NULL;
}
if (IS_ENABLED(CONFIG_ZONE_DMA) &&
dev->coherent_dma_mask <= DMA_BIT_MASK(32))
flags |= GFP_DMA;
if (IS_ENABLED(CONFIG_DMA_CMA) && (flags & __GFP_WAIT)) {
struct page *page;
void *addr;
size = PAGE_ALIGN(size);
page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
get_order(size));
if (!page)
return NULL;
*dma_handle = phys_to_dma(dev, page_to_phys(page));
addr = page_address(page);
memset(addr, 0, size);
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs) ||
dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs)) {
/*
* flush the caches here because we can't later
*/
__dma_flush_range(addr, addr + size);
__dma_remap(page, size, 0, true);
}
return addr;
} else {
return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
struct page *dma_alloc_at_from_contiguous(struct device *dev, int count,
unsigned int align, phys_addr_t at_addr)
{
unsigned long mask, pfn, pageno, start = 0;
struct cma *cma = dev_get_cma_area(dev);
struct page *page = NULL;
int ret;
unsigned long start_pfn = __phys_to_pfn(at_addr);
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,
count, align);
if (!count)
return NULL;
mask = (1 << align) - 1;
if (start_pfn && start_pfn < cma->base_pfn)
return NULL;
start = start_pfn ? start_pfn - cma->base_pfn : start;
mutex_lock(&cma_mutex);
for (;;) {
unsigned long timeout = jiffies + msecs_to_jiffies(8000);
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count || (start && start != pageno))
break;
pfn = cma->base_pfn + pageno;
retry:
ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
if (ret == 0) {
bitmap_set(cma->bitmap, pageno, count);
page = pfn_to_page(pfn);
break;
} else if (start && time_before(jiffies, timeout)) {
cond_resched();
invalidate_bh_lrus();
goto retry;
} else if (ret != -EBUSY || start) {
break;
}
pr_debug("%s(): memory range at %p is busy, retrying\n",
__func__, pfn_to_page(pfn));
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
mutex_unlock(&cma_mutex);
pr_debug("%s(): returned %p\n", __func__, page);
if (page) {
__dma_remap(page, count << PAGE_SHIFT,
pgprot_dmacoherent(PAGE_KERNEL));
__dma_clear_buffer(page, count << PAGE_SHIFT);
}
return page;
}
