/**
* dma_alloc_from_contiguous() - allocate pages from contiguous area
* @dev: Pointer to device for which the allocation is performed.
* @count: Requested number of pages.
* @align: Requested alignment of pages (in PAGE_SIZE order).
*
* This function allocates memory buffer for specified device. It uses
* device specific contiguous memory area if available or the default
* global one. Requires architecture specific get_dev_cma_area() helper
* function.
*/
struct page *dma_alloc_from_contiguous(struct device *dev, int count,
unsigned int align)
{
unsigned long mask, pfn, pageno, start = 0;
struct cma *cma = dev_get_cma_area(dev);
struct page *page = NULL;
int ret;
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,
count, align);
if (!count)
return NULL;
mask = (1 << align) - 1;
for (;;) {
mutex_lock(&cma->lock);
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count) {
mutex_unlock(&cma->lock);
break;
}
bitmap_set(cma->bitmap, pageno, count);
/*
* It's safe to drop the lock here. We've marked this region for
* our exclusive use. If the migration fails we will take the
* lock again and unmark it.
*/
mutex_unlock(&cma->lock);
pfn = cma->base_pfn + pageno;
mutex_lock(&cma_mutex);
ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
mutex_unlock(&cma_mutex);
if (ret == 0) {
page = pfn_to_page(pfn);
break;
} else if (ret != -EBUSY) {
clear_cma_bitmap(cma, pfn, count);
break;
}
clear_cma_bitmap(cma, pfn, count);
pr_debug("%s(): memory range at %p is busy, retrying\n",
__func__, pfn_to_page(pfn));
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
pr_debug("%s(): returned %p\n", __func__, page);
return page;
}
static struct page *__dma_alloc_from_contiguous(struct device *dev, int count,
unsigned int align)
{
unsigned long mask, pfn, pageno, start = 0;
struct cma *cma = dev_get_cma_area(dev);
int ret;
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,
count, align);
if (!count)
return NULL;
mask = (1 << align) - 1;
mutex_lock(&cma_mutex);
for (;;) {
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count) {
printk(KERN_ERR "%s : cma->count is %lu, "
"pageno is %lu\n", __func__,
cma->count, pageno);
ret = -ENOMEM;
goto error;
}
pfn = cma->base_pfn + pageno;
ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
if (ret == 0) {
bitmap_set(cma->bitmap, pageno, count);
break;
} else if (ret != -EBUSY && ret != -EAGAIN) {
goto error;
}
pr_debug("%s(): memory range at %p is busy, retrying\n",
__func__, pfn_to_page(pfn));
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
mutex_unlock(&cma_mutex);
pr_debug("%s(): returned %p\n", __func__, pfn_to_page(pfn));
return pfn_to_page(pfn);
error:
pr_err("%s(): returned error (%d)\n", __func__, ret);
mutex_unlock(&cma_mutex);
return NULL;
}
/**
* dma_alloc_from_contiguous() - allocate pages from contiguous area
* @dev: Pointer to device for which the allocation is performed.
* @count: Requested number of pages.
* @align: Requested alignment of pages (in PAGE_SIZE order).
*
* This function allocates memory buffer for specified device. It uses
* device specific contiguous memory area if available or the default
* global one. Requires architecture specific get_dev_cma_area() helper
* function.
*/
struct page *dma_alloc_from_contiguous(struct device *dev, int count,
unsigned int align)
{
unsigned long mask, pfn, pageno, start = 0;
struct cma *cma = dev_get_cma_area(dev);
struct page *page = NULL;
int ret;
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,
count, align);
if (!count)
return NULL;
mask = (1 << align) - 1;
/* HACK for MFC context buffer */
if (!strncmp(dev_name(dev), "ion_video", strlen("ion_video")) && (count > 8))
start = 16;
mutex_lock(&cma_mutex);
for (;;) {
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count)
break;
pfn = cma->base_pfn + pageno;
ret = cma->isolated ?
0 : alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
if (ret == 0) {
bitmap_set(cma->bitmap, pageno, count);
page = pfn_to_page(pfn);
cma->free_count -= count;
break;
} else if (ret != -EBUSY) {
break;
}
pr_debug("%s(): memory range at %p is busy, retrying\n",
__func__, pfn_to_page(pfn));
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
mutex_unlock(&cma_mutex);
pr_debug("%s(): returned %p\n", __func__, page);
return page;
}
/**
* dma_alloc_from_contiguous() - allocate pages from contiguous area
* @dev: Pointer to device for which the allocation is performed.
* @count: Requested number of pages.
* @align: Requested alignment of pages (in PAGE_SIZE order).
*
* This function allocates memory buffer for specified device. It uses
* device specific contiguous memory area if available or the default
* global one. Requires architecture specific get_dev_cma_area() helper
* function.
*/
struct page *dma_alloc_from_contiguous(struct device *dev, int count,
unsigned int align)
{
unsigned long mask, pfn = 0, pageno, start = 0;
struct cma *cma = dev_get_cma_area(dev);
struct page *page = NULL;
int ret;
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,
count, align);
if (!count)
return NULL;
mask = (1 << align) - 1;
mutex_lock(&cma_mutex);
for (;;) {
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count)
break;
pfn = cma->base_pfn + pageno;
ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
if (ret == 0) {
bitmap_set(cma->bitmap, pageno, count);
page = pfn_to_page(pfn);
break;
} else if (ret != -EBUSY) {
break;
}
pr_debug("%s(): memory range at %p (%lx, %lx) is busy,"\
" retrying\n", __func__, pfn_to_page(pfn),
pfn, pfn + count);
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
if (!page)
cma_bitmap_show(dev);
mutex_unlock(&cma_mutex);
pr_debug("%s(): returned %p pfn(%lx)\n", __func__, page, pfn);
return page;
}
/**
* dma_contiguous_isolate() - isolate contiguous memory from the page allocator
* @dev: Pointer to device which owns the contiguous memory
*
* This function isolates contiguous memory from the page allocator. If some of
* the contiguous memory is allocated, it is reclaimed.
*/
int dma_contiguous_isolate(struct device *dev)
{
struct cma *cma = dev_get_cma_area(dev);
int ret;
int idx;
if (!cma)
return -ENODEV;
if (cma->count == 0)
return 0;
mutex_lock(&cma_mutex);
if (cma->isolated) {
mutex_unlock(&cma_mutex);
dev_err(dev, "Alread isolated!\n");
return 0;
}
idx = find_first_zero_bit(cma->bitmap, cma->count);
while (idx < cma->count) {
int idx_set;
idx_set = find_next_bit(cma->bitmap, cma->count, idx);
do {
ret = alloc_contig_range(cma->base_pfn + idx,
cma->base_pfn + idx_set,
MIGRATE_CMA);
} while (ret == -EBUSY);
if (ret < 0) {
mutex_unlock(&cma_mutex);
dma_contiguous_deisolate_until(dev, idx_set);
dev_err(dev, "Failed to isolate %#lx@%#010llx (%d).\n",
(idx_set - idx) * PAGE_SIZE,
PFN_PHYS(cma->base_pfn + idx), ret);
return ret;
}
idx = find_next_zero_bit(cma->bitmap, cma->count, idx_set);
}
cma->isolated = true;
mutex_unlock(&cma_mutex);
return 0;
}
/**
* dma_alloc_from_contiguous() - allocate pages from contiguous area
* @dev: Pointer to device for which the allocation is performed.
* @count: Requested number of pages.
* @align: Requested alignment of pages (in PAGE_SIZE order).
*
* This function allocates memory buffer for specified device. It uses
* device specific contiguous memory area if available or the default
* global one. Requires architecture specific get_dev_cma_area() helper
* function.
*/
struct page *dma_alloc_from_contiguous(struct device *dev, int count,
unsigned int align)
{
unsigned long mask, pfn, pageno, start = 0;
struct cma *cma = dev_get_cma_area(dev);
struct page *page = NULL;
int ret;
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
if (!count)
return NULL;
mask = (1 << align) - 1;
mutex_lock(&cma_mutex);
for (;;) {
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count)
break;
pfn = cma->base_pfn + pageno;
ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
if (ret == 0) {
bitmap_set(cma->bitmap, pageno, count);
page = pfn_to_page(pfn);
break;
} else if (ret != -EBUSY) {
break;
}
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
mutex_unlock(&cma_mutex);
return page;
}
/**
* dma_alloc_from_contiguous() - allocate pages from contiguous area
* @dev: Pointer to device for which the allocation is performed.
* @count: Requested number of pages.
* @align: Requested alignment of pages (in PAGE_SIZE order).
*
* This function allocates memory buffer for specified device. It uses
* device specific contiguous memory area if available or the default
* global one. Requires architecture specific get_dev_cma_area() helper
* function.
*/
struct page *dma_alloc_from_contiguous(struct device *dev, int count,
unsigned int align)
{
unsigned long mask, pfn, pageno, start = 0;
unsigned long retry_timeout, retry_cnt;
struct cma *cma = dev_get_cma_area(dev);
struct page *page = NULL;
int ret;
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,
count, align);
if (!count)
return NULL;
retry_timeout = jiffies + msecs_to_jiffies(500);
retry_cnt = 0;
mask = (1 << align) - 1;
mutex_lock(&cma_mutex);
for (;;) {
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count) {
if (start == 0 ||
(time_is_before_jiffies(retry_timeout) && retry_cnt != 0))
break;
cond_resched();
retry_cnt++;
start = 0;
continue;
}
pfn = cma->base_pfn + pageno;
ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
if (ret == 0) {
bitmap_set(cma->bitmap, pageno, count);
page = pfn_to_page(pfn);
adjust_managed_cma_page_count(page_zone(page), -count);
break;
} else if (ret != -EBUSY) {
break;
}
pr_debug("%s(): memory range at %p is busy, retrying\n",
__func__, pfn_to_page(pfn));
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
mutex_unlock(&cma_mutex);
pr_debug("%s(): returned %p\n", __func__, page);
return page;
}
struct page *dma_alloc_at_from_contiguous(struct device *dev, int count,
unsigned int align, phys_addr_t at_addr)
{
unsigned long mask, pfn, pageno, start = 0;
struct cma *cma = dev_get_cma_area(dev);
struct page *page = NULL;
int ret;
unsigned long start_pfn = __phys_to_pfn(at_addr);
if (!cma || !cma->count)
return NULL;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,
count, align);
if (!count)
return NULL;
mask = (1 << align) - 1;
if (start_pfn && start_pfn < cma->base_pfn)
return NULL;
start = start_pfn ? start_pfn - cma->base_pfn : start;
mutex_lock(&cma_mutex);
for (;;) {
unsigned long timeout = jiffies + msecs_to_jiffies(8000);
pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
start, count, mask);
if (pageno >= cma->count || (start && start != pageno))
break;
pfn = cma->base_pfn + pageno;
retry:
ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
if (ret == 0) {
bitmap_set(cma->bitmap, pageno, count);
page = pfn_to_page(pfn);
break;
} else if (start && time_before(jiffies, timeout)) {
cond_resched();
invalidate_bh_lrus();
goto retry;
} else if (ret != -EBUSY || start) {
break;
}
pr_debug("%s(): memory range at %p is busy, retrying\n",
__func__, pfn_to_page(pfn));
/* try again with a bit different memory target */
start = pageno + mask + 1;
}
mutex_unlock(&cma_mutex);
pr_debug("%s(): returned %p\n", __func__, page);
if (page) {
__dma_remap(page, count << PAGE_SHIFT,
pgprot_dmacoherent(PAGE_KERNEL));
__dma_clear_buffer(page, count << PAGE_SHIFT);
}
return page;
}