/*
* @put_userptr: inform the allocator that a USERPTR buffer will no longer
* be used
*/
static void vb2_dma_sg_put_userptr(void *buf_priv)
{
struct vb2_dma_sg_buf *buf = buf_priv;
struct sg_table *sgt = &buf->sg_table;
int i = buf->num_pages;
DEFINE_DMA_ATTRS(attrs);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0)
dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
#endif
dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
__func__, buf->num_pages);
dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir,
&attrs);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(buf->dma_sgt);
while (--i >= 0) {
if (buf->dma_dir == DMA_FROM_DEVICE)
set_page_dirty_lock(buf->pages[i]);
if (!vma_is_io(buf->vma))
put_page(buf->pages[i]);
}
kfree(buf->pages);
vb2_put_vma(buf->vma);
kfree(buf);
}
static void vb2_dma_contig_put_userptr(void *mem_priv)
{
struct vb2_dc_buf *buf = mem_priv;
if (!buf)
return;
vb2_put_vma(buf->vma);
kfree(buf);
}
/*
* @put_userptr: inform the allocator that a USERPTR buffer will no longer
* be used
*/
static void vb2_dma_sg_put_userptr(void *buf_priv)
{
struct vb2_dma_sg_buf *buf = buf_priv;
int i = buf->num_pages;
dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
__func__, buf->num_pages);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(&buf->sg_table);
while (--i >= 0) {
if (buf->write)
set_page_dirty_lock(buf->pages[i]);
if (!vma_is_io(buf->vma))
put_page(buf->pages[i]);
}
kfree(buf->pages);
vb2_put_vma(buf->vma);
kfree(buf);
}
static void vb2_vmalloc_put_userptr(void *buf_priv)
{
struct vb2_vmalloc_buf *buf = buf_priv;
unsigned long vaddr = (unsigned long)buf->vaddr & PAGE_MASK;
unsigned int i;
if (buf->pages) {
if (vaddr)
vm_unmap_ram((void *)vaddr, buf->n_pages);
for (i = 0; i < buf->n_pages; ++i) {
if (buf->dma_dir == DMA_FROM_DEVICE)
set_page_dirty_lock(buf->pages[i]);
put_page(buf->pages[i]);
}
kfree(buf->pages);
} else {
vb2_put_vma(buf->vma);
iounmap((__force void __iomem *)buf->vaddr);
}
kfree(buf);
}
static void vb2_vmalloc_put_userptr(void *buf_priv)
{
struct vb2_vmalloc_buf *buf = buf_priv;
unsigned long vaddr = (unsigned long)buf->vaddr & PAGE_MASK;
unsigned int i;
if (buf->pages) {
if (vaddr)
vm_unmap_ram((void *)vaddr, buf->n_pages);
for (i = 0; i < buf->n_pages; ++i) {
if (buf->write)
set_page_dirty_lock(buf->pages[i]);
put_page(buf->pages[i]);
}
kfree(buf->pages);
} else {
if (buf->vma)
vb2_put_vma(buf->vma);
iounmap(buf->vaddr);
}
kfree(buf);
}
static void vb2_ion_put_userptr(void *mem_priv)
{
struct vb2_ion_buf *buf = mem_priv;
struct vb2_ion_conf *conf = buf->conf;
if (!buf) {
pr_err("No buffer to put\n");
return;
}
/* Unmap DVA, KVA */
if (conf->use_mmu)
iovmm_unmap(conf->dev, buf->dva);
ion_unmap_dma(conf->client, buf->handle);
if (buf->kva)
ion_unmap_kernel(conf->client, buf->handle);
ion_free(conf->client, buf->handle);
vb2_put_vma(buf->vma);
kfree(buf);
}
static void *vb2_dma_sg_get_userptr(void *alloc_ctx, unsigned long vaddr,
unsigned long size,
enum dma_data_direction dma_dir)
{
struct vb2_dma_sg_conf *conf = alloc_ctx;
struct vb2_dma_sg_buf *buf;
unsigned long first, last;
int num_pages_from_user;
struct vm_area_struct *vma;
struct sg_table *sgt;
DEFINE_DMA_ATTRS(attrs);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0)
dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
#endif
buf = kzalloc(sizeof *buf, GFP_KERNEL);
if (!buf)
return NULL;
buf->vaddr = NULL;
buf->dev = conf->dev;
buf->dma_dir = dma_dir;
buf->offset = vaddr & ~PAGE_MASK;
buf->size = size;
buf->dma_sgt = &buf->sg_table;
first = (vaddr & PAGE_MASK) >> PAGE_SHIFT;
last = ((vaddr + size - 1) & PAGE_MASK) >> PAGE_SHIFT;
buf->num_pages = last - first + 1;
buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),
GFP_KERNEL);
if (!buf->pages)
goto userptr_fail_alloc_pages;
vma = find_vma(current->mm, vaddr);
if (!vma) {
dprintk(1, "no vma for address %lu\n", vaddr);
goto userptr_fail_find_vma;
}
if (vma->vm_end < vaddr + size) {
dprintk(1, "vma at %lu is too small for %lu bytes\n",
vaddr, size);
goto userptr_fail_find_vma;
}
buf->vma = vb2_get_vma(vma);
if (!buf->vma) {
dprintk(1, "failed to copy vma\n");
goto userptr_fail_find_vma;
}
if (vma_is_io(buf->vma)) {
for (num_pages_from_user = 0;
num_pages_from_user < buf->num_pages;
++num_pages_from_user, vaddr += PAGE_SIZE) {
unsigned long pfn;
if (follow_pfn(vma, vaddr, &pfn)) {
dprintk(1, "no page for address %lu\n", vaddr);
break;
}
buf->pages[num_pages_from_user] = pfn_to_page(pfn);
}
} else
num_pages_from_user = get_user_pages(current, current->mm,
vaddr & PAGE_MASK,
buf->num_pages,
buf->dma_dir == DMA_FROM_DEVICE,
1, /* force */
buf->pages,
NULL);
if (num_pages_from_user != buf->num_pages)
goto userptr_fail_get_user_pages;
if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
buf->num_pages, buf->offset, size, 0))
goto userptr_fail_alloc_table_from_pages;
sgt = &buf->sg_table;
/*
* No need to sync to the device, this will happen later when the
* prepare() memop is called.
*/
sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
buf->dma_dir, &attrs);
if (!sgt->nents)
goto userptr_fail_map;
return buf;
userptr_fail_map:
sg_free_table(&buf->sg_table);
userptr_fail_alloc_table_from_pages:
userptr_fail_get_user_pages:
dprintk(1, "get_user_pages requested/got: %d/%d]\n",
buf->num_pages, num_pages_from_user);
if (!vma_is_io(buf->vma))
while (--num_pages_from_user >= 0)
put_page(buf->pages[num_pages_from_user]);
vb2_put_vma(buf->vma);
userptr_fail_find_vma:
kfree(buf->pages);
userptr_fail_alloc_pages:
kfree(buf);
return NULL;
}
static void *vb2_dma_sg_get_userptr(void *alloc_ctx, unsigned long vaddr,
unsigned long size, int write)
{
struct vb2_dma_sg_buf *buf;
unsigned long first, last;
int num_pages_from_user;
struct vm_area_struct *vma;
buf = kzalloc(sizeof *buf, GFP_KERNEL);
if (!buf)
return NULL;
buf->vaddr = NULL;
buf->write = write;
buf->offset = vaddr & ~PAGE_MASK;
buf->size = size;
first = (vaddr & PAGE_MASK) >> PAGE_SHIFT;
last = ((vaddr + size - 1) & PAGE_MASK) >> PAGE_SHIFT;
buf->num_pages = last - first + 1;
buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),
GFP_KERNEL);
if (!buf->pages)
goto userptr_fail_alloc_pages;
vma = find_vma(current->mm, vaddr);
if (!vma) {
dprintk(1, "no vma for address %lu\n", vaddr);
goto userptr_fail_find_vma;
}
if (vma->vm_end < vaddr + size) {
dprintk(1, "vma at %lu is too small for %lu bytes\n",
vaddr, size);
goto userptr_fail_find_vma;
}
buf->vma = vb2_get_vma(vma);
if (!buf->vma) {
dprintk(1, "failed to copy vma\n");
goto userptr_fail_find_vma;
}
if (vma_is_io(buf->vma)) {
for (num_pages_from_user = 0;
num_pages_from_user < buf->num_pages;
++num_pages_from_user, vaddr += PAGE_SIZE) {
unsigned long pfn;
if (follow_pfn(buf->vma, vaddr, &pfn)) {
dprintk(1, "no page for address %lu\n", vaddr);
break;
}
buf->pages[num_pages_from_user] = pfn_to_page(pfn);
}
} else
num_pages_from_user = get_user_pages(current, current->mm,
vaddr & PAGE_MASK,
buf->num_pages,
write,
1, /* force */
buf->pages,
NULL);
if (num_pages_from_user != buf->num_pages)
goto userptr_fail_get_user_pages;
if (sg_alloc_table_from_pages(&buf->sg_table, buf->pages,
buf->num_pages, buf->offset, size, 0))
goto userptr_fail_alloc_table_from_pages;
return buf;
userptr_fail_alloc_table_from_pages:
userptr_fail_get_user_pages:
dprintk(1, "get_user_pages requested/got: %d/%d]\n",
buf->num_pages, num_pages_from_user);
if (!vma_is_io(buf->vma))
while (--num_pages_from_user >= 0)
put_page(buf->pages[num_pages_from_user]);
vb2_put_vma(buf->vma);
userptr_fail_find_vma:
kfree(buf->pages);
userptr_fail_alloc_pages:
kfree(buf);
return NULL;
}